fix: eliminate v3/v4 double-counting, count assistant/message usage only

- findSessionFiles now picks newest schema version per session dir
  (v3 file is a migrated subset of v4; reading both doubled ~5.9k turns)
- handleLine only counts type=assistant/message with data.usage
  (compaction/summary re-reports summarizer spend separately)
- verified exact match vs independent ground truth: 16/16 days,
  all 13 model rows, grand total 3,725,441,198 tokens
This commit is contained in:
katR 2026-10-14 13:48:03 +05:30
parent 6ee09e215e
commit 60643d673c
5 changed files with 75 additions and 27 deletions

6
.gitignore vendored
View file

@ -1 +1,5 @@
node_modules/\n.DS_Store node_modules/
.DS_Store
# accuracy audit scripts (dev-only)
audit/

View file

@ -4,19 +4,35 @@ import os from "node:os";
import { spawn } from "node:child_process"; import { spawn } from "node:child_process";
const fileCache = /* @__PURE__ */ new Map(); const fileCache = /* @__PURE__ */ new Map();
function findSessionFiles(dir) { function findSessionFiles(dir) {
const results = []; const byDir = /* @__PURE__ */ new Map();
const collect = (d) => {
try { try {
const entries = fs.readdirSync(dir, { withFileTypes: true }); const entries = fs.readdirSync(d, { withFileTypes: true });
for (const entry of entries) { for (const entry of entries) {
const fullPath = path.join(dir, entry.name); const fullPath = path.join(d, entry.name);
if (entry.isDirectory()) { if (entry.isDirectory()) {
results.push(...findSessionFiles(fullPath)); collect(fullPath);
} else if (entry.name.endsWith(".jsonl.zstd") || entry.name.endsWith(".jsonl")) { } else if (entry.name.endsWith(".jsonl.zstd") || entry.name.endsWith(".jsonl")) {
results.push(fullPath); if (!byDir.has(d)) byDir.set(d, []);
byDir.get(d).push(fullPath);
} }
} }
} catch { } catch {
} }
};
collect(dir);
const results = [];
for (const files of byDir.values()) {
if (files.length === 1) {
results.push(files[0]);
continue;
}
const ranked = files.map((f) => {
const m = path.basename(f).match(/\.v(\d+)\.jsonl/);
return { f, v: m ? parseInt(m[1], 10) : 0 };
}).sort((a, b) => b.v - a.v);
results.push(ranked[0].f);
}
return results; return results;
} }
async function parseSessionFile(filePath) { async function parseSessionFile(filePath) {
@ -65,8 +81,9 @@ async function parseSessionFile(filePath) {
if (line.includes('"usage"')) { if (line.includes('"usage"')) {
try { try {
const obj = JSON.parse(line); const obj = JSON.parse(line);
if (obj.type !== "assistant/message") return;
const time = obj.time || obj.data && obj.data.time; const time = obj.time || obj.data && obj.data.time;
const usage = obj.usage || obj.data && obj.data.usage; const usage = obj.data && obj.data.usage || obj.usage;
if (time && usage) { if (time && usage) {
const d = new Date(time); const d = new Date(time);
const dateKey = `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, "0")}-${String(d.getDate()).padStart(2, "0")}`; const dateKey = `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, "0")}-${String(d.getDate()).padStart(2, "0")}`;

File diff suppressed because one or more lines are too long

View file

@ -1,6 +1,6 @@
{ {
"name": "token-contribution", "name": "token-contribution",
"version": "0.1.0", "version": "0.1.1",
"description": "GitHub-style token usage contribution calendar and analytics for DeepSeek Harness", "description": "GitHub-style token usage contribution calendar and analytics for DeepSeek Harness",
"type": "module", "type": "module",
"main": "./lib/index.js", "main": "./lib/index.js",

View file

@ -78,20 +78,43 @@ interface CacheEntry {
const fileCache = new Map<string, CacheEntry>(); const fileCache = new Map<string, CacheEntry>();
function findSessionFiles(dir: string): string[] { function findSessionFiles(dir: string): string[] {
const results: string[] = []; // Group files by session directory. DSH keeps a migrated copy of each
// session (session.v3.jsonl.zstd alongside session.v4.jsonl.zstd) that
// contains the SAME records — reading both double-counts every turn.
// Prefer the newest schema version per directory.
const byDir = new Map<string, string[]>();
const collect = (d: string) => {
try { try {
const entries = fs.readdirSync(dir, { withFileTypes: true }); const entries = fs.readdirSync(d, { withFileTypes: true });
for (const entry of entries) { for (const entry of entries) {
const fullPath = path.join(dir, entry.name); const fullPath = path.join(d, entry.name);
if (entry.isDirectory()) { if (entry.isDirectory()) {
results.push(...findSessionFiles(fullPath)); collect(fullPath);
} else if (entry.name.endsWith(".jsonl.zstd") || entry.name.endsWith(".jsonl")) { } else if (entry.name.endsWith(".jsonl.zstd") || entry.name.endsWith(".jsonl")) {
results.push(fullPath); if (!byDir.has(d)) byDir.set(d, []);
byDir.get(d)!.push(fullPath);
} }
} }
} catch { } catch {
// Ignore unreadable dirs // Ignore unreadable dirs
} }
};
collect(dir);
const results: string[] = [];
for (const files of byDir.values()) {
if (files.length === 1) {
results.push(files[0]);
continue;
}
// Prefer highest session version (v4 > v3)
const ranked = files
.map((f) => {
const m = path.basename(f).match(/\.v(\d+)\.jsonl/);
return { f, v: m ? parseInt(m[1], 10) : 0 };
})
.sort((a, b) => b.v - a.v);
results.push(ranked[0].f);
}
return results; return results;
} }
@ -140,8 +163,12 @@ async function parseSessionFile(filePath: string): Promise<SessionSummary | null
if (line.includes('"usage"')) { if (line.includes('"usage"')) {
try { try {
const obj = JSON.parse(line); const obj = JSON.parse(line);
// Only assistant/message records carry real billed usage.
// compaction/summary records re-report the summarizer call and
// assistant/attempt lines carry no data.usage at all.
if (obj.type !== "assistant/message") return;
const time = obj.time || (obj.data && obj.data.time); const time = obj.time || (obj.data && obj.data.time);
const usage = obj.usage || (obj.data && obj.data.usage); const usage = (obj.data && obj.data.usage) || obj.usage;
if (time && usage) { if (time && usage) {
const d = new Date(time); const d = new Date(time);
const dateKey = `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, "0")}-${String(d.getDate()).padStart(2, "0")}`; const dateKey = `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, "0")}-${String(d.getDate()).padStart(2, "0")}`;