Skip to content

Commit

Permalink
Unify traces description
Browse files Browse the repository at this point in the history 
Change showing device's id to grouping statistics by device's description.
Fixes Open-CAS/standalone-linux-io-tracer#166

Signed-off-by: Slawomir Jankowski <[email protected]>
  • Loading branch information
@Ostrokrzew
Ostrokrzew committed Mar 17, 2020
1 parent 821ffac commit da76ef5
Show file tree
Hide file tree
Showing 9 changed files with 366 additions and 232 deletions.
2 changes: 1 addition & 1 deletion source/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -156,4 +156,4 @@ file(APPEND ${CMAKE_BINARY_DIR}/octf-vars.cmake
# Install it next to octf-config
install(FILES ${CMAKE_BINARY_DIR}/octf-vars.cmake DESTINATION ${OCTF_PACKAGE_DIR} COMPONENT octf-install-cmake)

add_subdirectory(examples)
# add_subdirectory(examples)
2 changes: 2 additions & 0 deletions source/octf/analytics/statistics/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,4 +12,6 @@ PRIVATE
${CMAKE_CURRENT_LIST_DIR}/WorksetCalculatorDevices.cpp
${CMAKE_CURRENT_LIST_DIR}/FilesystemStatistics.h
${CMAKE_CURRENT_LIST_DIR}/FilesystemStatistics.cpp
${CMAKE_CURRENT_LIST_DIR}/FilesystemStatisticsEntry.h
${CMAKE_CURRENT_LIST_DIR}/FilesystemStatisticsEntry.cpp
)
272 changes: 73 additions & 199 deletions source/octf/analytics/statistics/FilesystemStatistics.cpp
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright(c) 2012-2018 Intel Corporation
* Copyright(c) 2012-2020 Intel Corporation
* SPDX-License-Identifier: BSD-3-Clause-Clear
*/

Expand All @@ -8,99 +8,96 @@
namespace octf {

struct FilesystemStatistics::Key {
Key(uint64_t id, uint64_t size, const std::string &name)
: Id(id)
, Size(size)
, Name(name) {}
Key(uint64_t id)
: Id(id)
, Size(0)
, Name("") {}
Key()
: name()
, devId()
, partId()
, statsCase(StatisticsCase::NAME_NOT_SET) {}

Key(StatisticsCase sCase,
const std::string &name,
uint64_t devId,
uint64_t partId)
: name(name)
, devId(devId)
, partId(partId)
, statsCase(sCase) {}

: Id()
, Size()
, Name("") {}
virtual ~Key() {}

Key(const Key &other)
: name(other.name)
, devId(other.devId)
, partId(other.partId)
, statsCase(other.statsCase) {}
: Id(other.Id)
, Size(other.Size)
, Name(other.Name) {}

Key &operator=(const Key &other) {
if (this != &other) {
name = other.name;
devId = other.devId;
partId = other.partId;
statsCase = other.statsCase;
Id = other.Id;
Size = other.Size;
Name = other.Name;
}

return *this;
}

bool operator==(const Key &other) const {
return name == other.name && devId == other.devId &&
partId == other.partId && statsCase == other.statsCase;
return Id == other.Id;
}

bool operator!=(const Key &other) const {
return !(*this == other);
return Id != other.Id;
}

bool operator<(const Key &other) const {
int result = name.compare(other.name);
if (0 == result) {
if (devId != other.devId) {
return devId < other.devId;
} else if (partId != other.partId) {
return partId < other.partId;
} else {
return statsCase < other.statsCase;
}
}

return result < 0;
return Id < other.Id;
}

std::string name;
uint64_t devId;
uint64_t partId;
StatisticsCase statsCase;
uint64_t Id;
uint64_t Size;
std::string Name;
};

FilesystemStatistics::FilesystemStatistics()
: m_children()
, m_ioStats()
, m_devId()
, m_partId()
, m_statsCase(StatisticsCase::NAME_NOT_SET) {}
: m_map() {}

FilesystemStatistics::~FilesystemStatistics() {}

FilesystemStatistics::FilesystemStatistics(const FilesystemStatistics &other)
: m_children(other.m_children)
, m_ioStats(other.m_ioStats)
, m_devId(other.m_devId)
, m_partId(other.m_partId)
, m_statsCase(other.m_statsCase) {}
: m_map(other.m_map) {}

FilesystemStatistics &FilesystemStatistics::operator=(
const FilesystemStatistics &other) {
if (&other != this) {
m_children = other.m_children;
m_ioStats = other.m_ioStats;
m_devId = other.m_devId;
m_partId = other.m_partId;
m_statsCase = other.m_statsCase;
m_map = other.m_map;
}

return *this;
}

void FilesystemStatistics::addDevice(
const proto::trace::EventDeviceDescription &devDesc) {
Key key(devDesc.id(), devDesc.size(), devDesc.name());

// Check maybe key already exists
auto iter = m_map.find(key);
if (iter != m_map.end()) {
// Key already exists, we have to update it, but need to keep the old
// copy of IO statistics

// Create pair of the updated key and the old statistics
auto pair = std::make_pair(Key(key), FilesystemStatisticsEntry(iter->second));

// Remove the old key
m_map.erase(key);

// Insert the new pair
auto result = m_map.emplace(pair);
if (!result.second || result.first == m_map.end()) {
throw Exception("Cannot allocate IO statistics");
}
} else {
// Key doesn't exist add it and allocate for them IO statistics
getStatisticsByKey(key);
}
}

void FilesystemStatistics::count(IFileSystemViewer *viewer,
const proto::trace::ParsedEvent &event) {
if (event.has_io()) {
Expand All @@ -115,176 +112,53 @@ void FilesystemStatistics::count(IFileSystemViewer *viewer,
const auto &device = event.device();

FileId id = FileId(event);
Key key(device.id());

auto &statistics = getStatisticsByIds(viewer, viewer->getParentId(id),
device.id());
statistics.updateIoStats(event);
getStatisticsByKey(key).updateIoStats(event);

{
// Update statistics by file extension
auto ext = viewer->getFileExtension(id);
if (ext != "") {
Key key(StatisticsCase::kFileExtension, ext, device.id(),
device.partition());
getStatisticsByKey(key).updateIoStats(event);
}
}
{
// Update statistics by base name
auto basename = viewer->getFileNamePrefix(id);
if (basename != "") {
Key key(StatisticsCase::kFileNamePrefix, basename, device.id(),
device.partition());
getStatisticsByKey(key).updateIoStats(event);
}
}
}
}

void FilesystemStatistics::getFilesystemStatistics(
proto::FilesystemStatistics *statistics) const {
fillProtoStatistics(statistics, "");
}

FilesystemStatistics &FilesystemStatistics::getStatisticsByIds(
IFileSystemViewer *viewer,
const FileId &dirId,
uint64_t devId) {
FilesystemStatistics *statistics = NULL;
FileId parentId = viewer->getParentId(dirId);

if (parentId == dirId) {
statistics = this;
} else {
statistics = &getStatisticsByIds(viewer, parentId, devId);
}

std::string name = viewer->getFileName(dirId);
Key key(StatisticsCase::kDirectory, name, devId, dirId.partitionId);

return statistics->getStatisticsByKey(key);
}

void FilesystemStatistics::fillProtoStatistics(
proto::FilesystemStatistics *statistics,
const std::string &dir) const {
proto::FilesystemStatisticsEntry entry;

if (dir != "") {
fillProtoStatisticsEntry(&entry, dir);

if (m_statsCase != StatisticsCase::kDirectory) {
auto &metrics = *entry.mutable_statistics()
->mutable_write()
->mutable_metrics();

if (metrics[WIF_METRIC_NAME].value() > 1.0L) {
// For non-directory stats, add only statistics which have write
// invalidation factor greater than 1.0. It minimize output
// size.
statistics->add_entries()->CopyFrom(entry);
}
} else {
statistics->add_entries()->CopyFrom(entry);
}
}

for (const auto &child : m_children) {
std::string name = child.first.name;
auto statsCase = child.first.statsCase;

if (name.empty()) {
continue;
}

if (statsCase == StatisticsCase::kDirectory) {
// Directory case
if (name.back() != '/' && !dir.empty() && dir.back() != '/') {
name = dir + "/" + name;
} else {
name = dir + name;
}
}

child.second.fillProtoStatistics(statistics, name);
}
}

void FilesystemStatistics::fillProtoStatisticsEntry(
proto::FilesystemStatisticsEntry *entry,
const std::string &name) const {
m_ioStats.getIoStatistics(entry->mutable_statistics());
entry->set_deviceid(m_devId);
entry->set_partitionid(m_partId);

// Clear proto statistics
auto pStats = entry->mutable_statistics();
pStats->clear_flush();
pStats->clear_discard();
pStats->mutable_read()->clear_latency();
pStats->mutable_write()->clear_latency();
pStats->mutable_total()->clear_latency();
pStats->mutable_read()->mutable_size()->clear_percentiles();
pStats->mutable_write()->mutable_size()->clear_percentiles();
pStats->mutable_total()->mutable_size()->clear_percentiles();

// Calculate WIF and insert metric
auto metrics = pStats->mutable_write()->mutable_metrics();
double workset = (*metrics)["workset"].value();
double written =
pStats->write().size().total() + pStats->discard().size().total();

double wif = 0.0L;
if (workset != 0.0L) {
wif = written / workset;
}

(*metrics)[WIF_METRIC_NAME].set_value(wif);

switch (m_statsCase) {
case StatisticsCase::kDirectory:
entry->set_directory(name);
break;
case StatisticsCase::kFileNamePrefix:
entry->set_filenameprefix(name);
break;
case StatisticsCase::kFileExtension:
entry->set_fileextension(name);
break;
default:
throw Exception("Invalid statistics case");
break;
void FilesystemStatistics::discard(const proto::trace::ParsedEvent &event) {
for (auto &child : m_map) {
child.second.discard(event);
}
}

void FilesystemStatistics::updateIoStats(
const proto::trace::ParsedEvent &event) {
m_ioStats.count(event);
void FilesystemStatistics::getFilesystemStatistics(
proto::FilesystemStatistics *statistics) const {
FilesystemStatisticsEntry entry;
entry.fillProtoStatistics(statistics, "");
}

void FilesystemStatistics::discard(const proto::trace::ParsedEvent &event) {
if (m_devId == event.device().id()) {
m_ioStats.count(event);
}
FilesystemStatisticsEntry &FilesystemStatistics::getStatisticsByKey(const Key &key) {
auto iter = m_map.find(key);
if (iter != m_map.end()) {
auto pair = std::make_pair(Key(key), FilesystemStatisticsEntry());

for (auto &child : m_children) {
child.second.discard(event);
}
}
auto result = m_map.emplace(pair);
if (!result.second || result.first == m_map.end()) {
throw Exception("Cannot allocate FS statistics");
}

FilesystemStatistics &FilesystemStatistics::getStatisticsByKey(const Key &key) {
auto iter = m_children.find(key);
if (iter != m_children.end()) {
return iter->second;
iter = result.first;
}

auto &newFsStats = m_children[key];

newFsStats.m_devId = key.devId;
newFsStats.m_partId = key.partId;
newFsStats.m_statsCase = key.statsCase;

return newFsStats;
return iter->second;
}

} /* namespace octf */
Loading

0 comments on commit da76ef5

Please sign in to comment.