llvm-project/compiler-rt/lib/tsan/rtl/tsan_sync.cpp

//===-- tsan_sync.cpp -----------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file is a part of ThreadSanitizer (TSan), a race detector.
//
//===----------------------------------------------------------------------===//
#include "sanitizer_common/sanitizer_placement_new.h"
#include "tsan_sync.h"
#include "tsan_rtl.h"
#include "tsan_mman.h"

namespace __tsan {

void DDMutexInit(ThreadState *thr, uptr pc, SyncVar *s);

SyncVar::SyncVar() : mtx(MutexTypeSyncVar) { Reset(); }

void SyncVar::Init(ThreadState *thr, uptr pc, uptr addr, bool save_stack) {
  Reset();
  this->addr = addr;
  next = 0;
  if (save_stack && !SANITIZER_GO)  // Go does not use them
    creation_stack_id = CurrentStackId(thr, pc);
  if (common_flags()->detect_deadlocks)
    DDMutexInit(thr, pc, this);
}

void SyncVar::Reset() {
  CHECK(!ctx->resetting);
  creation_stack_id = kInvalidStackID;
  owner_tid = kInvalidTid;
  last_lock.Reset();
  recursion = 0;
  atomic_store_relaxed(&flags, 0);
  Free(clock);
  Free(read_clock);
}

MetaMap::MetaMap()
    : block_alloc_("heap block allocator"), sync_alloc_("sync allocator") {}

void MetaMap::AllocBlock(ThreadState *thr, uptr pc, uptr p, uptr sz) {
  u32 idx = block_alloc_.Alloc(&thr->proc()->block_cache);
  MBlock *b = block_alloc_.Map(idx);
  b->siz = sz;
  b->tag = 0;
  b->tid = thr->tid;
  b->stk = CurrentStackId(thr, pc);
  u32 *meta = MemToMeta(p);
  DCHECK_EQ(*meta, 0);
  *meta = idx | kFlagBlock;
}

uptr MetaMap::FreeBlock(Processor *proc, uptr p, bool reset) {
  MBlock* b = GetBlock(p);
  if (b == 0)
    return 0;
  uptr sz = RoundUpTo(b->siz, kMetaShadowCell);
  FreeRange(proc, p, sz, reset);
  return sz;
}

bool MetaMap::FreeRange(Processor *proc, uptr p, uptr sz, bool reset) {
  bool has_something = false;
  u32 *meta = MemToMeta(p);
  u32 *end = MemToMeta(p + sz);
  if (end == meta)
    end++;
  for (; meta < end; meta++) {
    u32 idx = *meta;
    if (idx == 0) {
      // Note: don't write to meta in this case -- the block can be huge.
      continue;
    }
    *meta = 0;
    has_something = true;
    while (idx != 0) {
      if (idx & kFlagBlock) {
        block_alloc_.Free(&proc->block_cache, idx & ~kFlagMask);
        break;
      } else if (idx & kFlagSync) {
        DCHECK(idx & kFlagSync);
        SyncVar *s = sync_alloc_.Map(idx & ~kFlagMask);
        u32 next = s->next;
        if (reset)
          s->Reset();
        sync_alloc_.Free(&proc->sync_cache, idx & ~kFlagMask);
        idx = next;
      } else {
        CHECK(0);
      }
    }
  }
  return has_something;
}

// ResetRange removes all meta objects from the range.
// It is called for large mmap-ed regions. The function is best-effort wrt
// freeing of meta objects, because we don't want to page in the whole range
// which can be huge. The function probes pages one-by-one until it finds a page
// without meta objects, at this point it stops freeing meta objects. Because
// thread stacks grow top-down, we do the same starting from end as well.
void MetaMap::ResetRange(Processor *proc, uptr p, uptr sz, bool reset) {
  if (SANITIZER_GO) {
    // UnmapOrDie/MmapFixedNoReserve does not work on Windows,
    // so we do the optimization only for C/C++.
    FreeRange(proc, p, sz, reset);
    return;
  }
  const uptr kMetaRatio = kMetaShadowCell / kMetaShadowSize;
  const uptr kPageSize = GetPageSizeCached() * kMetaRatio;
  if (sz <= 4 * kPageSize) {
    // If the range is small, just do the normal free procedure.
    FreeRange(proc, p, sz, reset);
    return;
  }
  // First, round both ends of the range to page size.
  uptr diff = RoundUp(p, kPageSize) - p;
  if (diff != 0) {
    FreeRange(proc, p, diff, reset);
    p += diff;
    sz -= diff;
  }
  diff = p + sz - RoundDown(p + sz, kPageSize);
  if (diff != 0) {
    FreeRange(proc, p + sz - diff, diff, reset);
    sz -= diff;
  }
  // Now we must have a non-empty page-aligned range.
  CHECK_GT(sz, 0);
  CHECK_EQ(p, RoundUp(p, kPageSize));
  CHECK_EQ(sz, RoundUp(sz, kPageSize));
  const uptr p0 = p;
  const uptr sz0 = sz;
  // Probe start of the range.
  for (uptr checked = 0; sz > 0; checked += kPageSize) {
    bool has_something = FreeRange(proc, p, kPageSize, reset);
    p += kPageSize;
    sz -= kPageSize;
    if (!has_something && checked > (128 << 10))
      break;
  }
  // Probe end of the range.
  for (uptr checked = 0; sz > 0; checked += kPageSize) {
    bool has_something = FreeRange(proc, p + sz - kPageSize, kPageSize, reset);
    sz -= kPageSize;
    // Stacks grow down, so sync object are most likely at the end of the region
    // (if it is a stack). The very end of the stack is TLS and tsan increases
    // TLS by at least 256K, so check at least 512K.
    if (!has_something && checked > (512 << 10))
      break;
  }
  // Finally, page out the whole range (including the parts that we've just
  // freed). Note: we can't simply madvise, because we need to leave a zeroed
  // range (otherwise __tsan_java_move can crash if it encounters a left-over
  // meta objects in java heap).
  uptr metap = (uptr)MemToMeta(p0);
  uptr metasz = sz0 / kMetaRatio;
  UnmapOrDie((void*)metap, metasz);
  if (!MmapFixedSuperNoReserve(metap, metasz))
    Die();
}

void MetaMap::ResetClocks() {
  // This can be called from the background thread
  // which does not have proc/cache.
  // The cache is too large for stack.
  static InternalAllocatorCache cache;
  internal_memset(&cache, 0, sizeof(cache));
  internal_allocator()->InitCache(&cache);
  sync_alloc_.ForEach([&](SyncVar *s) {
    if (s->clock) {
      InternalFree(s->clock, &cache);
      s->clock = nullptr;
    }
    if (s->read_clock) {
      InternalFree(s->read_clock, &cache);
      s->read_clock = nullptr;
    }
    s->last_lock.Reset();
  });
  internal_allocator()->DestroyCache(&cache);
}

MBlock* MetaMap::GetBlock(uptr p) {
  u32 *meta = MemToMeta(p);
  u32 idx = *meta;
  for (;;) {
    if (idx == 0)
      return 0;
    if (idx & kFlagBlock)
      return block_alloc_.Map(idx & ~kFlagMask);
    DCHECK(idx & kFlagSync);
    SyncVar * s = sync_alloc_.Map(idx & ~kFlagMask);
    idx = s->next;
  }
}

SyncVar *MetaMap::GetSync(ThreadState *thr, uptr pc, uptr addr, bool create,
                          bool save_stack) {
  DCHECK(!create || thr->slot_locked);
  u32 *meta = MemToMeta(addr);
  u32 idx0 = *meta;
  u32 myidx = 0;
  SyncVar *mys = nullptr;
  for (;;) {
    for (u32 idx = idx0; idx && !(idx & kFlagBlock);) {
      DCHECK(idx & kFlagSync);
      SyncVar * s = sync_alloc_.Map(idx & ~kFlagMask);
      if (LIKELY(s->addr == addr)) {
        if (UNLIKELY(myidx != 0)) {
          mys->Reset();
          sync_alloc_.Free(&thr->proc()->sync_cache, myidx);
        }
        return s;
      }
      idx = s->next;
    }
    if (!create)
      return nullptr;
    if (UNLIKELY(*meta != idx0)) {
      idx0 = *meta;
      continue;
    }

    if (LIKELY(myidx == 0)) {
      myidx = sync_alloc_.Alloc(&thr->proc()->sync_cache);
      mys = sync_alloc_.Map(myidx);
      mys->Init(thr, pc, addr, save_stack);
    }
    mys->next = idx0;
    if (atomic_compare_exchange_strong((atomic_uint32_t*)meta, &idx0,
        myidx | kFlagSync, memory_order_release)) {
      return mys;
    }
  }
}

void MetaMap::MoveMemory(uptr src, uptr dst, uptr sz) {
  // src and dst can overlap,
  // there are no concurrent accesses to the regions (e.g. stop-the-world).
  CHECK_NE(src, dst);
  CHECK_NE(sz, 0);

  // The current MoveMemory implementation behaves incorrectly when src, dst,
  // and sz are not aligned to kMetaShadowCell.
  // For example, with kMetaShadowCell == 8:
  // - src = 4: unexpectedly clears the metadata for the range [0, 4).
  // - src = 16, dst = 4, size = 8: A sync variable for addr = 20, which should
  //   be moved to the metadata for address 8, is incorrectly moved to the
  //   metadata for address 0 instead.
  // - src = 0, sz = 4: fails to move the tail metadata.
  // Therefore, the following assertions is needed.
  DCHECK_EQ(src % kMetaShadowCell, 0);
  DCHECK_EQ(dst % kMetaShadowCell, 0);
  DCHECK_EQ(sz % kMetaShadowCell, 0);

  uptr diff = dst - src;
  u32 *src_meta, *dst_meta, *src_meta_end;
  uptr inc;
  if (dst < src) {
    src_meta = MemToMeta(src);
    dst_meta = MemToMeta(dst);
    src_meta_end = MemToMeta(src + sz);
    inc = 1;
  } else {
    src_meta = MemToMeta(src + sz) - 1;
    dst_meta = MemToMeta(dst + sz) - 1;
    src_meta_end = MemToMeta(src) - 1;
    inc = -1;
  }
  for (; src_meta != src_meta_end; src_meta += inc, dst_meta += inc) {
    CHECK_EQ(*dst_meta, 0);
    u32 idx = *src_meta;
    *src_meta = 0;
    *dst_meta = idx;
    // Patch the addresses in sync objects.
    while (idx != 0) {
      if (idx & kFlagBlock)
        break;
      CHECK(idx & kFlagSync);
      SyncVar *s = sync_alloc_.Map(idx & ~kFlagMask);
      s->addr += diff;
      idx = s->next;
    }
  }
}

void MetaMap::OnProcIdle(Processor *proc) {
  block_alloc_.FlushCache(&proc->block_cache);
  sync_alloc_.FlushCache(&proc->sync_cache);
}

MetaMap::MemoryStats MetaMap::GetMemoryStats() const {
  MemoryStats stats;
  stats.mem_block = block_alloc_.AllocatedMemory();
  stats.sync_obj = sync_alloc_.AllocatedMemory();
  return stats;
}

}  // namespace __tsan