v0.1.3: Optimize upload performance with gzip compression

cmd/tokitoki/main_test.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"compress/gzip"
 	"encoding/json"
 	"net/http"
 	"net/http/httptest"
@@ -63,7 +64,7 @@ func TestRunSetKeyWritesAPIKey(t *testing.T) {
 		t.Fatalf("run(set key) = %d, want 0", code)
 	}
 
-	path := filepath.Join(home, config.DataDirName, "api_key")
+	path := filepath.Join(home, config.DataDirName, "config", "api_key")
 	data, err := os.ReadFile(path)
 	if err != nil {
 		t.Fatal(err)
@@ -86,14 +87,6 @@ func TestRunGetKeyReturnsErrorWhenMissing(t *testing.T) {
 	if code := run([]string{"get", "key"}); code != 1 {
 		t.Fatalf("run(get key) = %d, want 1", code)
 	}
-	path := filepath.Join(home, config.DataDirName, "api_key")
-	data, err := os.ReadFile(path)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if string(data) != "" {
-		t.Fatalf("api_key = %q, want empty file", string(data))
-	}
 }
 
 func TestRunGetKeyRejectsExtraArgs(t *testing.T) {
@@ -136,14 +129,25 @@ func TestRunHeartbeatUploadsUnifiedIDEEvent(t *testing.T) {
 		if r.URL.Path != "/api/usage-events/batch" {
 			t.Errorf("path = %q, want usage batch endpoint", r.URL.Path)
 		}
-		if err := json.NewDecoder(r.Body).Decode(&payload); err != nil {
+		// Handle gzip-compressed request bodies.
+		body := r.Body
+		if r.Header.Get("Content-Encoding") == "gzip" {
+			gr, err := gzip.NewReader(r.Body)
+			if err != nil {
+				t.Error(err)
+				return
+			}
+			defer gr.Close()
+			body = gr
+		}
+		if err := json.NewDecoder(body).Decode(&payload); err != nil {
 			t.Error(err)
 		}
 		accepted := []string{}
 		if len(payload.Events) == 1 {
 			accepted = append(accepted, payload.Events[0].ID)
 		}
-		_ = json.NewEncoder(w).Encode(usageupload.Response{OK: true, Accepted: accepted})
+		_ = json.NewEncoder(w).Encode(usageupload.Response{OK: true, Accepted: accepted, Duplicate: []string{}, Rejected: []usageupload.Reject{}})
 	}))
 	defer server.Close()
 	t.Setenv(usageupload.BaseURLEnv, server.URL)
@@ -208,14 +212,25 @@ func TestRunHeartbeatAppliesProjectIdentityFile(t *testing.T) {
 
 	var payload usageupload.Payload
 	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if err := json.NewDecoder(r.Body).Decode(&payload); err != nil {
+		// Handle gzip-compressed request bodies.
+		body := r.Body
+		if r.Header.Get("Content-Encoding") == "gzip" {
+			gr, err := gzip.NewReader(r.Body)
+			if err != nil {
+				t.Error(err)
+				return
+			}
+			defer gr.Close()
+			body = gr
+		}
+		if err := json.NewDecoder(body).Decode(&payload); err != nil {
 			t.Error(err)
 		}
 		accepted := []string{}
 		if len(payload.Events) == 1 {
 			accepted = append(accepted, payload.Events[0].ID)
 		}
-		_ = json.NewEncoder(w).Encode(usageupload.Response{OK: true, Accepted: accepted})
+		_ = json.NewEncoder(w).Encode(usageupload.Response{OK: true, Accepted: accepted, Duplicate: []string{}, Rejected: []usageupload.Reject{}})
 	}))
 	defer server.Close()
 	t.Setenv(usageupload.BaseURLEnv, server.URL)

internal/agentusage/providers.go

@@ -1,10 +1,19 @@
 package agentusage
 
 import (
+	"sort"
+
 	"github.com/tokitoki-dev/tokitoki-cli/internal/usage"
 	"github.com/tokitoki-dev/tokitoki-cli/internal/usageprovider"
 )
 
+func sortEntriesByTimestampDesc(entries []usage.Entry) []usage.Entry {
+	sort.Slice(entries, func(i, j int) bool {
+		return entries[i].Timestamp.After(entries[j].Timestamp)
+	})
+	return entries
+}
+
 // providerBase carries the scan configuration shared by every agent
 // provider. filter skips source files whose events are already ingested;
 // the SQLite-backed providers (Kilo, Hermes, Goose) deliberately do not
@@ -89,9 +98,13 @@ func (p CopilotProvider) WithFileFilter(filter usage.FileFilter) usageprovider.P
 	return p
 }
 
-// Entries loads normalized GitHub Copilot CLI usage entries.
+// Entries loads normalized GitHub Copilot CLI usage entries, newest first.
 func (p CopilotProvider) Entries() ([]usage.Entry, error) {
-	return loadCopilotEntries(p.paths, p.filter)
+	entries, err := loadCopilotEntries(p.paths, p.filter)
+	if err != nil {
+		return nil, err
+	}
+	return sortEntriesByTimestampDesc(entries), nil
 }
 
 // WithPaths returns a Gemini CLI provider configured with data roots.
@@ -109,9 +122,13 @@ func (p GeminiProvider) WithFileFilter(filter usage.FileFilter) usageprovider.Pr
 	return p
 }
 
-// Entries loads normalized Gemini CLI usage entries.
+// Entries loads normalized Gemini CLI usage entries, newest first.
 func (p GeminiProvider) Entries() ([]usage.Entry, error) {
-	return loadGeminiEntries(p.paths, p.filter)
+	entries, err := loadGeminiEntries(p.paths, p.filter)
+	if err != nil {
+		return nil, err
+	}
+	return sortEntriesByTimestampDesc(entries), nil
 }
 
 // WithPaths returns a Kimi provider configured with data roots.
@@ -129,9 +146,13 @@ func (p KimiProvider) WithFileFilter(filter usage.FileFilter) usageprovider.Prov
 	return p
 }
 
-// Entries loads normalized Kimi usage entries.
+// Entries loads normalized Kimi usage entries, newest first.
 func (p KimiProvider) Entries() ([]usage.Entry, error) {
-	return loadKimiEntries(p.paths, p.filter)
+	entries, err := loadKimiEntries(p.paths, p.filter)
+	if err != nil {
+		return nil, err
+	}
+	return sortEntriesByTimestampDesc(entries), nil
 }
 
 // WithPaths returns a Qwen provider configured with data roots.
@@ -149,9 +170,13 @@ func (p QwenProvider) WithFileFilter(filter usage.FileFilter) usageprovider.Prov
 	return p
 }
 
-// Entries loads normalized Qwen usage entries.
+// Entries loads normalized Qwen usage entries, newest first.
 func (p QwenProvider) Entries() ([]usage.Entry, error) {
-	return loadQwenEntries(p.paths, p.filter)
+	entries, err := loadQwenEntries(p.paths, p.filter)
+	if err != nil {
+		return nil, err
+	}
+	return sortEntriesByTimestampDesc(entries), nil
 }
 
 // WithPaths returns an OpenClaw provider configured with data roots.
@@ -169,9 +194,13 @@ func (p OpenClawProvider) WithFileFilter(filter usage.FileFilter) usageprovider.
 	return p
 }
 
-// Entries loads normalized OpenClaw usage entries.
+// Entries loads normalized OpenClaw usage entries, newest first.
 func (p OpenClawProvider) Entries() ([]usage.Entry, error) {
-	return loadOpenClawEntries(p.paths, p.filter)
+	entries, err := loadOpenClawEntries(p.paths, p.filter)
+	if err != nil {
+		return nil, err
+	}
+	return sortEntriesByTimestampDesc(entries), nil
 }
 
 // WithPaths returns a pi-agent provider configured with data roots.
@@ -189,9 +218,13 @@ func (p PiProvider) WithFileFilter(filter usage.FileFilter) usageprovider.Provid
 	return p
 }
 
-// Entries loads normalized pi-agent usage entries.
+// Entries loads normalized pi-agent usage entries, newest first.
 func (p PiProvider) Entries() ([]usage.Entry, error) {
-	return loadPiEntries(p.paths, p.filter)
+	entries, err := loadPiEntries(p.paths, p.filter)
+	if err != nil {
+		return nil, err
+	}
+	return sortEntriesByTimestampDesc(entries), nil
 }
 
 // WithPaths returns an Amp provider configured with data roots.
@@ -209,9 +242,13 @@ func (p AmpProvider) WithFileFilter(filter usage.FileFilter) usageprovider.Provi
 	return p
 }
 
-// Entries loads normalized Amp usage entries.
+// Entries loads normalized Amp usage entries, newest first.
 func (p AmpProvider) Entries() ([]usage.Entry, error) {
-	return loadAmpEntries(p.paths, p.filter)
+	entries, err := loadAmpEntries(p.paths, p.filter)
+	if err != nil {
+		return nil, err
+	}
+	return sortEntriesByTimestampDesc(entries), nil
 }
 
 // WithPaths returns a Droid provider configured with data roots.
@@ -229,9 +266,13 @@ func (p DroidProvider) WithFileFilter(filter usage.FileFilter) usageprovider.Pro
 	return p
 }
 
-// Entries loads normalized Droid usage entries.
+// Entries loads normalized Droid usage entries, newest first.
 func (p DroidProvider) Entries() ([]usage.Entry, error) {
-	return loadDroidEntries(p.paths, p.filter)
+	entries, err := loadDroidEntries(p.paths, p.filter)
+	if err != nil {
+		return nil, err
+	}
+	return sortEntriesByTimestampDesc(entries), nil
 }
 
 // WithPaths returns a Kilo provider configured with data roots.
@@ -242,9 +283,13 @@ func (KiloProvider) WithPaths(paths []string) usageprovider.Provider {
 // Provider returns the Kilo provider id.
 func (KiloProvider) Provider() usage.Provider { return usage.ProviderKilo }
 
-// Entries loads normalized Kilo usage entries.
+// Entries loads normalized Kilo usage entries, newest first.
 func (p KiloProvider) Entries() ([]usage.Entry, error) {
-	return loadKiloEntries(p.paths)
+	entries, err := loadKiloEntries(p.paths)
+	if err != nil {
+		return nil, err
+	}
+	return sortEntriesByTimestampDesc(entries), nil
 }
 
 // WithPaths returns a Hermes provider configured with data roots.
@@ -255,9 +300,13 @@ func (HermesProvider) WithPaths(paths []string) usageprovider.Provider {
 // Provider returns the Hermes provider id.
 func (HermesProvider) Provider() usage.Provider { return usage.ProviderHermes }
 
-// Entries loads normalized Hermes Agent usage entries.
+// Entries loads normalized Hermes Agent usage entries, newest first.
 func (p HermesProvider) Entries() ([]usage.Entry, error) {
-	return loadHermesEntries(p.paths)
+	entries, err := loadHermesEntries(p.paths)
+	if err != nil {
+		return nil, err
+	}
+	return sortEntriesByTimestampDesc(entries), nil
 }
 
 // WithPaths returns a Codebuff provider configured with data roots.
@@ -275,9 +324,13 @@ func (p CodebuffProvider) WithFileFilter(filter usage.FileFilter) usageprovider.
 	return p
 }
 
-// Entries loads normalized Codebuff usage entries.
+// Entries loads normalized Codebuff usage entries, newest first.
 func (p CodebuffProvider) Entries() ([]usage.Entry, error) {
-	return loadCodebuffEntries(p.paths, p.filter)
+	entries, err := loadCodebuffEntries(p.paths, p.filter)
+	if err != nil {
+		return nil, err
+	}
+	return sortEntriesByTimestampDesc(entries), nil
 }
 
 // WithPaths returns an OpenCode provider configured with data roots.
@@ -296,9 +349,13 @@ func (p OpenCodeProvider) WithFileFilter(filter usage.FileFilter) usageprovider.
 	return p
 }
 
-// Entries loads normalized OpenCode usage entries.
+// Entries loads normalized OpenCode usage entries, newest first.
 func (p OpenCodeProvider) Entries() ([]usage.Entry, error) {
-	return loadOpenCodeEntries(p.paths, p.filter)
+	entries, err := loadOpenCodeEntries(p.paths, p.filter)
+	if err != nil {
+		return nil, err
+	}
+	return sortEntriesByTimestampDesc(entries), nil
 }
 
 // WithPaths returns a Goose provider configured with data roots.
@@ -309,7 +366,11 @@ func (GooseProvider) WithPaths(paths []string) usageprovider.Provider {
 // Provider returns the Goose provider id.
 func (GooseProvider) Provider() usage.Provider { return usage.ProviderGoose }
 
-// Entries loads normalized Goose usage entries.
+// Entries loads normalized Goose usage entries, newest first.
 func (p GooseProvider) Entries() ([]usage.Entry, error) {
-	return loadGooseEntries(p.paths)
+	entries, err := loadGooseEntries(p.paths)
+	if err != nil {
+		return nil, err
+	}
+	return sortEntriesByTimestampDesc(entries), nil
 }

internal/claudeusage/loader.go

@@ -756,3 +756,4 @@ func pathParts(path string) []string {
 	})
 	return parts
 }
+

internal/claudeusage/provider.go

@@ -1,6 +1,8 @@
 package claudeusage
 
 import (
+	"sort"
+
 	"github.com/tokitoki-dev/tokitoki-cli/internal/usage"
 	"github.com/tokitoki-dev/tokitoki-cli/internal/usageprovider"
 )
@@ -31,11 +33,15 @@ func (Provider) Provider() usage.Provider {
 	return usage.ProviderClaude
 }
 
-// Entries loads normalized Claude usage entries.
+// Entries loads normalized Claude usage entries, newest first.
 func (p Provider) Entries() ([]usage.Entry, error) {
 	entries, err := LoadEntriesFromPaths(p.paths, "", p.filter)
 	if err != nil {
 		return nil, err
 	}
-	return ConvertEntries(entries), nil
+	converted := ConvertEntries(entries)
+	sort.Slice(converted, func(i, j int) bool {
+		return converted[i].Timestamp.After(converted[j].Timestamp)
+	})
+	return converted, nil
 }

internal/codexusage/provider.go

@@ -1,6 +1,8 @@
 package codexusage
 
 import (
+	"sort"
+
 	"github.com/tokitoki-dev/tokitoki-cli/internal/usage"
 	"github.com/tokitoki-dev/tokitoki-cli/internal/usageprovider"
 )
@@ -31,7 +33,14 @@ func (Provider) Provider() usage.Provider {
 	return usage.ProviderCodex
 }
 
-// Entries loads normalized Codex usage entries.
+// Entries loads normalized Codex usage entries, newest first.
 func (p Provider) Entries() ([]usage.Entry, error) {
-	return LoadEntriesFromPaths(p.paths, "", p.filter)
+	entries, err := LoadEntriesFromPaths(p.paths, "", p.filter)
+	if err != nil {
+		return nil, err
+	}
+	sort.Slice(entries, func(i, j int) bool {
+		return entries[i].Timestamp.After(entries[j].Timestamp)
+	})
+	return entries, nil
 }

internal/store/lock.go

@@ -32,10 +32,14 @@ func AcquireDataLock(dir string, timeout time.Duration) (*DataLock, error) {
 	return lock, nil
 }
 
-// AcquireLock takes an exclusive advisory lock on dir/name, waiting up to
+// AcquireLock takes an exclusive advisory lock on dir/state/name, waiting up to
 // timeout before giving up with ErrLockBusy.
 func AcquireLock(dir, name string, timeout time.Duration) (*DataLock, error) {
-	path := filepath.Join(dir, name)
+	stateDir := filepath.Join(dir, "state")
+	if err := os.MkdirAll(stateDir, 0o700); err != nil {
+		return nil, fmt.Errorf("create state dir: %w", err)
+	}
+	path := filepath.Join(stateDir, name)
 	file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o600)
 	if err != nil {
 		return nil, fmt.Errorf("open lock file: %w", err)