fully automate seq_latency js creation

assets/benchmarking-dwarfs/js/seq_latency.js

@@ -1,52 +1,46 @@
-let ctx = document.getElementById("seq_read_latency_chart");
+const labels = ['Null 25 GiB file', 'Random 25 GiB file', '100 million-sided polygon data', 'Linux LTS kernel', '1024 null files', '1024 random files']
-const labels = [
-  "Null 25 GiB file",
-  "Random 25 GiB file",
-  "100 million-sided polygon data",
-  "Linux LTS kernel",
-];
 let data = [
         {
-    label: "DwarFS",
+        label: 'DwarFS',
-    data: [0.37114600000000003, 14.15143, 2.95083, 0.001523],
+        data: [0.37114600000000003, 14.15143, 2.95083, 0.001523, 0.014287000000000001, 0.013595000000000001],
-    backgroundColor: "rgb(255, 99, 132)",
+        backgroundColor: 'rgb(255, 99, 132)',
         },
-  {
-    label: "fuse-archive (tar)",
-    data: [0.393568, 0.397626, 0.07750499999999999, 0.0012230000000000001],
-    backgroundColor: "rgb(75, 192, 192)",
-  },
-  {
-    label: "Btrfs",
-    data: [
-      0.027922000000000002, 0.290906, 0.14088399999999998,
-      0.0013930000000000001,
-    ],
-    backgroundColor: "rgb(54, 162, 235)",
-  },
-];
         
-let config = {
+        {
-  type: "bar",
+        label: 'fuse-archive (tar)',
+        data: [0.393568, 0.397626, 0.07750499999999999, 0.0012230000000000001, 0.013405, 0.013465],
+        backgroundColor: 'rgb(75, 192, 192)',
+        },
+        
+        {
+        label: 'Btrfs',
+        data: [0.027922000000000002, 0.290906, 0.14088399999999998, 0.0013930000000000001, 0.0032860000000000003, 0.003326],
+        backgroundColor: 'rgb(54, 162, 235)',
+        },
+        
+]
+
+    let config = {
+        type: 'bar',
         data: {
             datasets: data,
-    labels,
+            labels
         },
         options: {
         plugins: {
             title: {
             display: true,
-        text: "Sequential Read Latency - in ms",
+            text: 'Sequential Read Latency - in ms'
             },
         },
         responsive: true,
         interaction: {
             intersect: false,
         },
-  },
+        }
-};
+    };
     
-Chart.defaults.borderColor = "#eee";
+Chart.defaults.borderColor = "#eee"
 Chart.defaults.color = "#eee";
-
+let ctx = document.getElementById("seq_read_latency_chart");
 new Chart(ctx, config);

assets/benchmarking-dwarfs/process-data.py

@@ -2,6 +2,10 @@
 import csv
 import re
 
+
+# a bunch of horrible code to make the chart.js code
+
+
 class HelperFunctions:
     def get_fs(dir):
         if dir.endswith('dwarfs'):
@@ -20,6 +24,10 @@ class HelperFunctions:
             return '100 million-sided polygon data'
         elif filename.startswith('kernel'):
             return 'Linux LTS kernel'
+        elif filename == 'small-files/random':
+            return '1024 random files'
+        elif filename == 'small-files/null':
+            return '1024 null files'
 
     def convert_time(time: str, unit: str) -> int:
         unit_exponents = ['ns', 'µs', 'ms', 's']
@@ -33,15 +41,18 @@ class HelperFunctions:
         else:
             current_unit = 's'
 
-        unit_multiplier = unit_exponents.index(current_unit) - unit_exponents.index(unit)
+        unit_multiplier = unit_exponents.index(current_unit) - unit_exponents.index(
-        return HelperFunctions.time_int(time) * (1000 ** unit_multiplier)
+            unit
+        )
+        return HelperFunctions.time_num(time) * (1000**unit_multiplier)
 
-    def time_int(time: str):
+    def time_num(time: str):
-        time = re.sub("[^0-9\\.]", "", time)
+        time = re.sub('[^0-9\\.]', '', time)
         return float(time)
 
 
-def sequential_latency():
+def get_seq_latency_data() -> tuple:
+    # format: { 'labels': ['btrfs'], 'btrfs': [9, 8, 4, 6]}
     datasets = {'labels': []}
     with open('assets/benchmarking-dwarfs/data/benchmark-data.csv', 'rt') as f:
         for line in csv.reader(f):
@@ -56,18 +67,68 @@ def sequential_latency():
                 datasets[fs] = []
                 datasets[fs].append(line[3])
 
-    return datasets
+    # NOTE: this will break if the bulk data contains a larger unit than the single file data, but that's unlikely to happen so I'm not gonna deal with it
+    largest_time_unit = 'ns'
+    for key in datasets.keys():
+        if key == 'labels':
+            continue
+        for item in datasets[key]:
+            if item.endswith('ms'):
+                largest_time_unit = 'ms'
+            elif item.endswith('µs') and largest_time_unit != 'ms':
+                largest_time_unit = 'µs'
+            elif (
+                item.endswith('ns')
+                and largest_time_unit != 'ms'
+                and largest_time_unit != 'µs'
+            ):
+                largest_time_unit = 'ns'
+            elif re.sub('[0-9]', '', item) == 's':
+                largest_time_unit = 's'
+                break
 
+    for key in datasets.keys():
+        if key == 'labels':
+            continue
+        for i in range(len(datasets[key])):
+            datasets[key][i] = HelperFunctions.convert_time(
+                datasets[key][i], largest_time_unit
+            )
 
-def singles():
+    with open('assets/benchmarking-dwarfs/data/bulk.csv', 'rt') as f:
-    pass
+        for line in csv.reader(f):
+            if line[2] != 'bulk_sequential_read_latency':
+                continue
+            fs = HelperFunctions.get_fs(line[0])
+            label = HelperFunctions.get_label(line[1])
+            datasets['labels'].append(label) if label not in datasets[
+                'labels'
+            ] else False
 
+            for item in line[3:]:
+                if item.endswith('ms'):
+                    largest_time_unit = 'ms'
+                elif item.endswith('µs') and largest_time_unit != 'ms':
+                    largest_time_unit = 'µs'
+                elif (
+                    item.endswith('ns')
+                    and largest_time_unit != 'ms'
+                    and largest_time_unit != 'µs'
+                ):
+                    largest_time_unit = 'ns'
+                elif re.sub('[0-9]', '', item) == 's':
+                    largest_time_unit = 's'
+                    break
 
-def bulk():
+            for i in range(len(line[3:])):
-    pass
+                line[i + 3] = HelperFunctions.convert_time(item, largest_time_unit)
 
+            datasets[fs].append(sum(line[3:]) / len(line[3:]))
 
-if __name__ == '__main__':
+        return (datasets, largest_time_unit)
+
+def seq_latency():
+    with open('assets/benchmarking-dwarfs/js/seq_latency.js', 'wt') as f:
         # from https://github.com/chartjs/Chart.js/blob/master/docs/scripts/utils.js (CHART_COLORS)
         # modified so similar color aren't adjacent
         chart_colors = [
@@ -80,17 +141,18 @@ if __name__ == '__main__':
             "'rgb(201, 203, 207)'",  # grey
         ]
 
-    print('Sequential latency:')
+        #print('Sequential latency:')
         labels_code = 'const labels = $labels$'
         dataset_code = '''
         {
         label: '$label$',
         data: $data$,
         backgroundColor: $color$,
-    },'''
+        },
+        '''
 
         config_code = '''
-let config = {
+    let config = {
         type: 'bar',
         data: {
             datasets: data,
@@ -109,38 +171,39 @@ let config = {
         },
         }
     };
-'''
+    '''
 
-    data = sequential_latency()
+        data, largest_time_unit = get_seq_latency_data()
         labels_code = labels_code.replace('$labels$', format(data['labels']))
-    print(labels_code)
+        f.write(labels_code)
         data.pop('labels')
-    print('let data = [', end='')
+        f.write('\nlet data = [')
-    largest_time_unit = 'ns'
         for fs in data.keys():
-        for item in data[fs]:
+            f.write(
-            if item.endswith('ms'):
-                largest_time_unit = 'ms'
-            elif item.endswith('µs') and largest_time_unit != 'ms':
-                largest_time_unit = 'µs'
-            elif item.endswith('ns') and largest_time_unit != 'ms' and largest_time_unit != 'µs':
-                largest_time_unit = 'ns'
-            elif re.sub('[0-9]', '', item) == 's':
-                largest_time_unit = 's'
-                break
-
-        for i in range(len(data[fs])):
-            data[fs][i] = HelperFunctions.convert_time(data[fs][i], largest_time_unit)
-        
-        print(
                 dataset_code.replace('$label$', fs)
                 .replace('$data$', format(data[fs]))
-            .replace('$color$', format(chart_colors[list(data.keys()).index(fs)])),
+                .replace('$color$', format(chart_colors[list(data.keys()).index(fs)]))
-            end=''
             )
-    print('\n]\n')
+        f.write('\n]\n')
 
         title = 'Sequential Read Latency'
-    print(config_code.replace('$title$', title).replace('$timeunit$', largest_time_unit))
+        f.write(
-    print('\nChart.defaults.borderColor = "#eee"')
+            config_code.replace('$title$', title).replace('$timeunit$', largest_time_unit)
-    print('Chart.defaults.color = "#eee";')
+        )
+
+        f.write('\nChart.defaults.borderColor = "#eee"\n')
+        f.write('Chart.defaults.color = "#eee";\n')
+        f.write('let ctx = document.getElementById("seq_read_latency_chart");\n')
+        f.write('new Chart(ctx, config);\n')
+
+
+def singles():
+    pass
+
+
+def bulk():
+    pass
+
+
+if __name__ == '__main__':
+    seq_latency()

blog/benchmarking-dwarfs.html

@@ -126,7 +126,6 @@
         </ol>
         <h2 id="footnotes">Footnotes</h2>
         <!-- JavaScript for graphs goes hereeeeeee -->
-        <!-- EXAMPLE HERE -->
         <script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
         <script src="/assets/benchmarking-dwarfs/js/seq_latency.js"></script>
         <section id="footnotes"
@@ -145,7 +144,8 @@
         generates regular polygons and writes their data to a file. I
         chose this because it was an artificial and reproducible yet
         fairly compressible dataset (without being extremely
-        compressible like null data).
+        compressible like null data).<br />
+
         <details open>
         <summary>
         3-sided regular polygon data