0.1.10

.github/workflows/lint.yaml

@@ -0,0 +1,21 @@
+name: Python Linting
+
+on: [push, pull_request]
+
+jobs:
+  PythonLinting:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.11'
+    - name: Install dependencies
+      run: |
+        pip install poetry
+        poetry install
+    - name: Run black
+      run: |
+        poetry run black --check .

bespoke-dataset-viewer/app/globals.css

@@ -34,6 +34,9 @@ body {
     --chart-5: 27 87% 67%;
     --radius: 0.5rem;
     --success: 142 40% 75%;
+    --tooltip-bg: white;
+    --tooltip-text-bg: rgb(255, 255, 255);
+    --tooltip-text: black;
   }
   .dark {
     --background: 0 0% 3.9%;
@@ -61,6 +64,10 @@ body {
     --chart-4: 280 65% 60%;
     --chart-5: 340 75% 55%;
     --success: 142 72% 40%; /* Slightly brighter green for dark mode */
+    --tooltip-bg: rgb(90, 90, 90);
+    --tooltip-text-bg: rgb(0, 0, 0);
+    --tooltip-text: rgb(250, 250, 250);
+    --border: rgb(39, 39, 42);
   }
 }
 

bespoke-dataset-viewer/components/dataset-viewer/DatasetViewer.tsx

@@ -18,12 +18,14 @@ import { FileText, Loader2, RefreshCcw } from "lucide-react"
 import { useCallback, useEffect, useState } from "react"
 import { DetailsSidebar } from "./DetailsSidebar"
 import { DistributionChart } from "./DistributionChart"
+import { TimeSeriesChart } from './TimeSeriesChart'
 
 const COLUMNS: Column[] = [
   { key: "user_message", label: "User Message" },
   { key: "assistant_message", label: "Assistant Message" },
   { key: "prompt_tokens", label: "Prompt Tokens" },
-  { key: "completion_tokens", label: "Completion Tokens" }
+  { key: "completion_tokens", label: "Completion Tokens" },
+  { key: "generation_time", label: "Generation Time (s)" }
 ]
 
 interface DatasetViewerProps {
@@ -61,7 +63,7 @@ export function DatasetViewer({ runHash, batchMode }: DatasetViewerProps) {
   const [data, setData] = useState<DataItem[]>([])
   const [theme, setTheme] = useState<"light" | "dark">("light")
   const [mounted, setMounted] = useState(false)
-  const [selectedDistribution, setSelectedDistribution] = useState<string | null>("total_tokens")
+  const [selectedDistribution, setSelectedDistribution] = useState<string | null>("requests")
   const [selectedItem, setSelectedItem] = useState<DataItem | null>(null)
   const { toast } = useToast()
   const [isPolling, setIsPolling] = useState(false)
@@ -224,29 +226,37 @@ export function DatasetViewer({ runHash, batchMode }: DatasetViewerProps) {
               <p className="text-sm text-muted-foreground">View and analyze your dataset responses</p>
             </div>
             {data.length > 0 && (
-              <DropdownMenu>
-                <DropdownMenuTrigger asChild>
-                  <Button variant="outline">
-                    {selectedDistribution
-                      ? selectedDistribution.split('_').map(word =>
-                        word.charAt(0).toUpperCase() + word.slice(1)
-                      ).join(' ')
-                      : 'Select Metric'}
-                  </Button>
-                </DropdownMenuTrigger>
-                <DropdownMenuContent align="end">
-                  <DropdownMenuItem onClick={() => setSelectedDistribution(null)}>
-                    None
-                  </DropdownMenuItem>
-                  {["total_tokens", "prompt_tokens", "completion_tokens"].map((column) => (
-                    <DropdownMenuItem key={column} onClick={() => setSelectedDistribution(column)}>
-                      {column === "total_tokens" ? "Total Tokens" :
-                        column === "prompt_tokens" ? "Prompt Tokens" :
-                          "Completion Tokens"}
+              <div className="flex gap-2">
+                <DropdownMenu>
+                  <DropdownMenuTrigger asChild>
+                    <Button variant="outline">
+                      {selectedDistribution
+                        ? selectedDistribution === "requests" 
+                          ? "Requests & Responses"
+                          : selectedDistribution.split('_').map(word =>
+                              word.charAt(0).toUpperCase() + word.slice(1)
+                            ).join(' ')
+                        : 'Select Metric'}
+                    </Button>
+                  </DropdownMenuTrigger>
+                  <DropdownMenuContent align="end">
+                    <DropdownMenuItem onClick={() => setSelectedDistribution(null)}>
+                      None
                     </DropdownMenuItem>
-                  ))}
-                </DropdownMenuContent>
-              </DropdownMenu>
+                    <DropdownMenuItem onClick={() => setSelectedDistribution("requests")}>
+                      Requests & Responses
+                    </DropdownMenuItem>
+                    {["total_tokens", "prompt_tokens", "completion_tokens", "generation_time"].map((column) => (
+                      <DropdownMenuItem key={column} onClick={() => setSelectedDistribution(column)}>
+                        {column === "total_tokens" ? "Total Tokens" :
+                          column === "prompt_tokens" ? "Prompt Tokens" :
+                            column === "completion_tokens" ? "Completion Tokens" :
+                              "Generation Time (s)"}
+                      </DropdownMenuItem>
+                    ))}
+                  </DropdownMenuContent>
+                </DropdownMenu>
+              </div>
             )}
           </div>
 
@@ -264,10 +274,14 @@ export function DatasetViewer({ runHash, batchMode }: DatasetViewerProps) {
               <div className="mb-8 space-y-4">
                 {selectedDistribution && (
                   <div className="rounded-lg border bg-card p-4">
-                    <DistributionChart
-                      data={data}
-                      column={selectedDistribution}
-                    />
+                    {selectedDistribution === "requests" ? (
+                      <TimeSeriesChart data={data} />
+                    ) : (
+                      <DistributionChart
+                        data={data}
+                        column={selectedDistribution}
+                      />
+                    )}
                   </div>
                 )}
               </div>

bespoke-dataset-viewer/components/dataset-viewer/DetailsSidebar.tsx

@@ -120,6 +120,13 @@ export function DetailsSidebar({ item, onClose }: DetailsSidebarProps) {
                   </div>
                 </div>
               </div>
+              <Separator />
+              <div className="space-y-2">
+                <h3 className="text-lg font-semibold">Generation Time</h3>
+                <p className="text-2xl font-bold">
+                  {((new Date(item.finished_at).getTime() - new Date(item.created_at).getTime()) / 1000).toFixed(2)}s
+                </p>
+              </div>
             </CardContent>
           </div>
         </div>

bespoke-dataset-viewer/components/dataset-viewer/DistributionChart.tsx

@@ -1,6 +1,7 @@
 "use client"
 
 import { BarChart, Bar, XAxis, YAxis, Tooltip, ResponsiveContainer } from 'recharts'
+import { bin } from 'd3-array'
 import { DataItem } from '@/types/dataset'
 
 interface DistributionChartProps {
@@ -11,43 +12,8 @@ interface DistributionChartProps {
 const COLUMN_DISPLAY_NAMES: Record<string, string> = {
   total_tokens: "Total Tokens",
   prompt_tokens: "Prompt Tokens",
-  completion_tokens: "Completion Tokens"
-}
-
-function calculateNiceBinSize(min: number, max: number): { binSize: number, start: number, end: number } {
-  // Calculate the range
-  const range = max - min;
-
-  // Get the magnitude of the range (in powers of 10)
-  const magnitude = Math.floor(Math.log10(range));
-
-  // Possible bin sizes (1, 2, 5, 10) * 10^n
-  const base = Math.pow(10, magnitude);
-  const possibleBinSizes = [
-    0.1 * base,
-    0.2 * base,
-    0.5 * base,
-    base,
-    2 * base,
-    5 * base,
-    10 * base
-  ];
-
-  // Aim for between 5-10 bins
-  const targetBins = 7;
-
-  // Find the bin size that gives us closest to our target number of bins
-  const binSize = possibleBinSizes.reduce((prev, curr) => {
-    const prevBins = range / prev;
-    const currBins = range / curr;
-    return Math.abs(currBins - targetBins) < Math.abs(prevBins - targetBins) ? curr : prev;
-  });
-
-  // Calculate nice start and end points
-  const start = Math.floor(min / binSize) * binSize;
-  const end = Math.ceil(max / binSize) * binSize;
-
-  return { binSize, start, end };
+  completion_tokens: "Completion Tokens",
+  generation_time: "Generation Time (s)"
 }
 
 function getDistributionData(data: DataItem[], column: string) {
@@ -59,53 +25,25 @@ function getDistributionData(data: DataItem[], column: string) {
         return item.raw_response.usage?.prompt_tokens || item.raw_response.response?.body?.usage?.prompt_tokens || 0;
       case "completion_tokens":
         return item.raw_response.usage?.completion_tokens || item.raw_response.response?.body?.usage?.completion_tokens || 0;
+      case "generation_time":
+        return (new Date(item.finished_at).getTime() - new Date(item.created_at).getTime()) / 1000;
       default:
         return 0;
     }
   });
 
-  const min = Math.min(...values);
-  const max = Math.max(...values);
+  const binner = bin()
+    .value(d => d)
+    .thresholds(10);
 
-  // If min equals max, all values are the same
-  if (min === max) {
-    return [{
-      range: `${min}`,
-      count: values.length
-    }];
-  }
-
-  const { binSize, start, end } = calculateNiceBinSize(min, max);
-  const numBins = Math.ceil((end - start) / binSize);
-
-  // Initialize buckets
-  const buckets: { range: string; start: number; count: number }[] = [];
-  for (let i = 0; i < numBins; i++) {
-    const bucketStart = start + (i * binSize);
-    const bucketEnd = bucketStart + binSize;
-    buckets.push({
-      range: `${Math.round(bucketStart)}-${Math.round(bucketEnd)}`,
-      start: bucketStart,
-      count: 0
-    });
-  }
-
-  // Count values in buckets
-  values.forEach(value => {
-    const bucketIndex = Math.floor((value - start) / binSize);
-    if (bucketIndex >= 0 && bucketIndex < buckets.length) {
-      buckets[bucketIndex].count++;
-    }
-  });
+  const bins = binner(values);
 
-  // Only return buckets that have counts
-  return buckets
-    .filter(bucket => bucket.count > 0)
-    .sort((a, b) => a.start - b.start)
-    .map(({ range, count }) => ({
-      range,
-      count
-    }));
+  return bins.map(b => ({
+    range: Number.isInteger(b.x0) ? `${b.x0}` : `${b.x0?.toFixed(2)}`,
+    count: b.length,
+    start: b.x0,
+    end: b.x1
+  }));
 }
 
 export function DistributionChart({ data, column }: DistributionChartProps) {
@@ -133,6 +71,18 @@ export function DistributionChart({ data, column }: DistributionChartProps) {
           <YAxis />
           <Tooltip 
             formatter={(value: number) => [`Count: ${value}`, displayName]}
+            contentStyle={{
+              backgroundColor: 'var(--tooltip-text-bg)',
+              border: '1px solid var(--border)',
+              borderRadius: '4px',
+              color: 'var(--tooltip-text)',
+              boxShadow: '0 2px 4px rgba(0,0,0,0.1)'
+            }}
+            labelStyle={{
+              color: 'var(--tooltip-text)',
+              marginBottom: '4px'
+            }}
+            cursor={{ fill: 'var(--tooltip-bg)', opacity: 0.2 }}
           />
           <Bar 
             dataKey="count"