diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 984c7fc50..3fef76939 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -39,7 +39,9 @@ "GOTMPDIR": "/home/vscode/.local/share/tmp", "PGDATA": "/home/vscode/.local/share/pg/pgdata", "PGPORT": "55432", - "PGSOCK_DIR": "/home/vscode/.local/share/pg" + "PGSOCK_DIR": "/home/vscode/.local/share/pg", + "FOREST_JWT": "${localEnv:FOREST_JWT}", + "SINGULARITY_TEST_EXTERNAL_API": "${localEnv:SINGULARITY_TEST_EXTERNAL_API}" }, "runArgs": ["--userns=keep-id"], "containerUser": "vscode" diff --git a/.github/workflows/devcontainer-podman.yml b/.github/workflows/devcontainer-podman.yml index f70f3a99d..e3e1b8152 100644 --- a/.github/workflows/devcontainer-podman.yml +++ b/.github/workflows/devcontainer-podman.yml @@ -54,6 +54,9 @@ jobs: disable-docker: true cache-storage: true + # containerEnv in devcontainer.json pulls these via ${localEnv:...} at + # container start; scope the secret to this step only. FOREST_JWT is + # empty on fork PRs (GitHub policy); SkipIfNotExternalAPI catches that. - name: Build devcontainer id: build uses: parkan/github-actions/devcontainer-build@v3 @@ -61,6 +64,9 @@ jobs: workspace-folder: . container-runtime: podman container-id-label: ci=podman + env: + FOREST_JWT: ${{ secrets.FOREST_JWT }} + SINGULARITY_TEST_EXTERNAL_API: 'true' - name: Generate swagger code if: steps.codegen.outputs.needed == 'true' @@ -103,7 +109,8 @@ jobs: with: container-id: ${{ steps.build.outputs.container-id }} command: > - cd /workspaces/singularity && mkdir -p artifacts && make test GOTESTSUM_ARGS="--junitfile artifacts/unit-tests.xml" + cd /workspaces/singularity && mkdir -p artifacts && + make test GOTESTSUM_ARGS="--junitfile artifacts/unit-tests.xml" container-runtime: podman - name: Run integration tests diff --git a/service/dealpusher/dealpusher_test.go b/service/dealpusher/dealpusher_test.go index 210871359..de7bdb3eb 100644 --- a/service/dealpusher/dealpusher_test.go +++ b/service/dealpusher/dealpusher_test.go @@ -126,7 +126,7 @@ func (m *MockDealMaker) MakeDeal(ctx context.Context, actorObj model.Actor, car func TestDealMakerService_Start(t *testing.T) { testutil.All(t, func(ctx context.Context, t *testing.T, db *gorm.DB) { - service, err := NewDealPusher(db, testutil.TestLotusAPI, "", 1, 10) + service, err := NewDealPusher(db, testutil.TestLotusAPI, testutil.TestLotusToken, 1, 10) require.NoError(t, err) ctx, cancel := context.WithCancel(ctx) exitErr := make(chan error, 1) @@ -140,9 +140,9 @@ func TestDealMakerService_Start(t *testing.T) { func TestDealMakerService_MultipleInstances(t *testing.T) { testutil.All(t, func(ctx context.Context, t *testing.T, db *gorm.DB) { - service1, err := NewDealPusher(db, testutil.TestLotusAPI, "", 1, 10) + service1, err := NewDealPusher(db, testutil.TestLotusAPI, testutil.TestLotusToken, 1, 10) require.NoError(t, err) - service2, err := NewDealPusher(db, testutil.TestLotusAPI, "", 1, 10) + service2, err := NewDealPusher(db, testutil.TestLotusAPI, testutil.TestLotusToken, 1, 10) require.NoError(t, err) ctx, cancel := context.WithTimeout(ctx, time.Second) defer cancel() @@ -161,7 +161,7 @@ func TestDealMakerService_FailtoSend(t *testing.T) { waitPendingInterval = time.Minute }() testutil.All(t, func(ctx context.Context, t *testing.T, db *gorm.DB) { - service, err := NewDealPusher(db, testutil.TestLotusAPI, "", 2, 0) + service, err := NewDealPusher(db, testutil.TestLotusAPI, testutil.TestLotusToken, 2, 0) require.NoError(t, err) mockDealmaker := new(MockDealMaker) service.dealMaker = mockDealmaker @@ -215,7 +215,7 @@ func TestDealMakerService_Cron(t *testing.T) { waitPendingInterval = time.Minute }() testutil.All(t, func(ctx context.Context, t *testing.T, db *gorm.DB) { - service, err := NewDealPusher(db, testutil.TestLotusAPI, "", 1, 10) + service, err := NewDealPusher(db, testutil.TestLotusAPI, testutil.TestLotusToken, 1, 10) require.NoError(t, err) mockDealmaker := new(MockDealMaker) service.dealMaker = mockDealmaker @@ -312,7 +312,7 @@ func TestDealMakerService_ScheduleWithConstraints(t *testing.T) { waitPendingInterval = time.Minute }() testutil.All(t, func(ctx context.Context, t *testing.T, db *gorm.DB) { - service, err := NewDealPusher(db, testutil.TestLotusAPI, "", 1, 10) + service, err := NewDealPusher(db, testutil.TestLotusAPI, testutil.TestLotusToken, 1, 10) require.NoError(t, err) mockDealmaker := new(MockDealMaker) service.dealMaker = mockDealmaker @@ -421,7 +421,7 @@ func TestDealMakerService_ScheduleWithConstraints(t *testing.T) { func TestDealmakerService_Force(t *testing.T) { testutil.All(t, func(ctx context.Context, t *testing.T, db *gorm.DB) { - service, err := NewDealPusher(db, testutil.TestLotusAPI, "", 1, 10) + service, err := NewDealPusher(db, testutil.TestLotusAPI, testutil.TestLotusToken, 1, 10) require.NoError(t, err) mockDealmaker := new(MockDealMaker) service.dealMaker = mockDealmaker @@ -480,7 +480,7 @@ func TestDealmakerService_Force(t *testing.T) { func TestDealMakerService_MaxReplica(t *testing.T) { testutil.All(t, func(ctx context.Context, t *testing.T, db *gorm.DB) { - service, err := NewDealPusher(db, testutil.TestLotusAPI, "", 1, 1) + service, err := NewDealPusher(db, testutil.TestLotusAPI, testutil.TestLotusToken, 1, 1) require.NoError(t, err) mockDealmaker := new(MockDealMaker) service.dealMaker = mockDealmaker @@ -537,7 +537,7 @@ func TestDealMakerService_MaxReplica(t *testing.T) { func TestDealMakerService_NewScheduleOneOff(t *testing.T) { testutil.All(t, func(ctx context.Context, t *testing.T, db *gorm.DB) { - service, err := NewDealPusher(db, testutil.TestLotusAPI, "", 1, 10) + service, err := NewDealPusher(db, testutil.TestLotusAPI, testutil.TestLotusToken, 1, 10) require.NoError(t, err) mockDealmaker := new(MockDealMaker) service.dealMaker = mockDealmaker diff --git a/service/dealtracker/ddo_tracking_test.go b/service/dealtracker/ddo_tracking_test.go index f14d31b8a..d99a23606 100644 --- a/service/dealtracker/ddo_tracking_test.go +++ b/service/dealtracker/ddo_tracking_test.go @@ -202,7 +202,7 @@ func TestRunOnce_DoesNotEpochExpireDDODeals(t *testing.T) { url, server := setupTestServerWithBody(t, `{}`) defer server.Close() - tracker := NewDealTracker(db, 0, url, testutil.TestLotusAPI, "", true) + tracker := NewDealTracker(db, 0, url, testutil.TestLotusAPI, testutil.TestLotusToken, true) require.NoError(t, tracker.runOnce(ctx)) var ddoActive model.Deal diff --git a/service/dealtracker/dealtracker.go b/service/dealtracker/dealtracker.go index 2e6a1ea5a..15c5dcdc1 100644 --- a/service/dealtracker/dealtracker.go +++ b/service/dealtracker/dealtracker.go @@ -442,7 +442,9 @@ func (d *DealTracker) runOnce(ctx context.Context) error { return errors.Wrapf(err, "failed to get lotus head time from %s", d.lotusURL) } - var lastEpoch int32 + // derive from head so expiration works even if the market stream is empty; + // the trackDeal callback can still bump it higher if a deal reports newer + lastEpoch := int32(epochutil.TimeToEpoch(headTime)) db := d.dbNoContext.WithContext(ctx) var actors []model.Actor diff --git a/service/dealtracker/dealtracker_test.go b/service/dealtracker/dealtracker_test.go index b36b9c244..7e23699c9 100644 --- a/service/dealtracker/dealtracker_test.go +++ b/service/dealtracker/dealtracker_test.go @@ -338,7 +338,7 @@ func TestRunOnce(t *testing.T) { url, server := setupTestServerWithBody(t, string(body)) defer server.Close() require.NoError(t, err) - tracker := NewDealTracker(db, time.Minute, url, testutil.TestLotusAPI, "", true) + tracker := NewDealTracker(db, time.Minute, url, testutil.TestLotusAPI, testutil.TestLotusToken, true) err = tracker.runOnce(context.Background()) require.NoError(t, err) var allDeals []model.Deal @@ -403,7 +403,7 @@ func TestRunOnce_DoesNotEpochExpirePDPDeals(t *testing.T) { url, server := setupTestServerWithBody(t, `{}`) defer server.Close() - tracker := NewDealTracker(db, time.Minute, url, testutil.TestLotusAPI, "", true) + tracker := NewDealTracker(db, time.Minute, url, testutil.TestLotusAPI, testutil.TestLotusToken, true) require.NoError(t, tracker.runOnce(ctx)) var marketDeal model.Deal diff --git a/service/epochutil/epoch_test.go b/service/epochutil/epoch_test.go index 65bec75fa..0d2298f03 100644 --- a/service/epochutil/epoch_test.go +++ b/service/epochutil/epoch_test.go @@ -9,20 +9,20 @@ import ( ) func TestDefaultValue(t *testing.T) { - err := Initialize(context.Background(), testutil.TestLotusAPI, "") + err := Initialize(context.Background(), testutil.TestLotusAPI, testutil.TestLotusToken) require.NoError(t, err) require.EqualValues(t, 1598306400, GenesisTimestamp) } func TestCalibNet(t *testing.T) { // This test may fail when calibnet resets - err := Initialize(context.Background(), "https://api.calibration.node.glif.io/rpc/v1", "") + err := Initialize(context.Background(), testutil.CalibnetRPC, testutil.TestLotusToken) require.NoError(t, err) require.EqualValues(t, 1667326380, GenesisTimestamp) } func TestEpochToTime(t *testing.T) { - err := Initialize(context.Background(), testutil.TestLotusAPI, "") + err := Initialize(context.Background(), testutil.TestLotusAPI, testutil.TestLotusToken) require.NoError(t, err) require.EqualValues(t, 1598306400, GenesisTimestamp) require.EqualValues(t, 1598306400, EpochToTime(0).Unix()) @@ -30,7 +30,7 @@ func TestEpochToTime(t *testing.T) { } func TestUnixToEpoch(t *testing.T) { - err := Initialize(context.Background(), testutil.TestLotusAPI, "") + err := Initialize(context.Background(), testutil.TestLotusAPI, testutil.TestLotusToken) require.NoError(t, err) require.EqualValues(t, 1598306400, GenesisTimestamp) require.EqualValues(t, 0, UnixToEpoch(1598306400)) @@ -38,7 +38,7 @@ func TestUnixToEpoch(t *testing.T) { } func TestTimeToEpoch(t *testing.T) { - err := Initialize(context.Background(), testutil.TestLotusAPI, "") + err := Initialize(context.Background(), testutil.TestLotusAPI, testutil.TestLotusToken) require.NoError(t, err) require.EqualValues(t, 1598306400, GenesisTimestamp) require.EqualValues(t, 0, TimeToEpoch(EpochToTime(0))) diff --git a/util/testutil/anvil.go b/util/testutil/anvil.go index bf5c10cbf..961923a05 100644 --- a/util/testutil/anvil.go +++ b/util/testutil/anvil.go @@ -41,11 +41,18 @@ func StartAnvil(t *testing.T, upstreamRPC string) *AnvilInstance { t.Fatalf("finding free port: %v", err) } - cmd := exec.Command("anvil", + args := []string{ "--fork-url", upstreamRPC, "--port", fmt.Sprintf("%d", port), "--silent", - ) + } + // forest gateway needs bearer auth; anvil supports --fork-header + haveTok := os.Getenv("FOREST_JWT") != "" + if haveTok { + args = append(args, "--fork-header", "Authorization: Bearer "+os.Getenv("FOREST_JWT")) + } + t.Logf("anvil forking %s (auth=%t)", upstreamRPC, haveTok) + cmd := exec.Command("anvil", args...) cmd.Stdout = os.Stderr cmd.Stderr = os.Stderr if err := cmd.Start(); err != nil { diff --git a/util/testutil/calibnet.go b/util/testutil/calibnet.go index d95d12f50..b15b76edb 100644 --- a/util/testutil/calibnet.go +++ b/util/testutil/calibnet.go @@ -1,8 +1,11 @@ package testutil // calibnet constants shared by integration tests + +// CalibnetRPC defaults to the self-hosted forest gateway; glif rate-limited CI. +var CalibnetRPC = envOr("SINGULARITY_TEST_CALIBNET_RPC", "https://static.187.115.235.167.clients.your-server.de/calibnet/rpc/v1") + const ( - CalibnetRPC = "https://api.calibration.node.glif.io/rpc/v1" CalibnetChainID = 314159 CalibnetDDOContract = "0x889fD50196BE300D06dc4b8F0F17fdB0af587095" CalibnetPaymentsContract = "0x09a0fDc2723fAd1A7b8e3e00eE5DF73841df55a0" diff --git a/util/testutil/testutils.go b/util/testutil/testutils.go index 8955e69a3..efa82d595 100644 --- a/util/testutil/testutils.go +++ b/util/testutil/testutils.go @@ -55,18 +55,37 @@ const TestWalletAddr = "f1fib3pv7jua2ockdugtz7viz3cyy6lkhh7rfx3sa" const TestPrivateKeyHex = "7b2254797065223a22736563703235366b31222c22507269766174654b6579223a226b35507976337148327349586343595a58594f5775453149326e32554539436861556b6c4e36695a5763453d227d" -// TestLotusAPI is the Lotus API endpoint to use for tests -// Using /rpc/v1 (stable) instead of deprecated /rpc/v0 -const TestLotusAPI = "https://api.node.glif.io/rpc/v1" +// TestLotusAPI defaults to the self-hosted forest gateway; glif rate-limited CI. +var TestLotusAPI = envOr("SINGULARITY_TEST_LOTUS_API", "https://static.187.115.235.167.clients.your-server.de/rpc/v1") -// SkipIfNotExternalAPI skips the test if SINGULARITY_TEST_EXTERNAL_API is not set -// Use this for tests that make external API calls (e.g., Lotus/Filecoin APIs) -// These are skipped by default because external APIs may be unreliable or rate-limited +// TestLotusToken is the bearer token for the forest gateway; empty in local +// dev and on fork PRs (see SkipIfNotExternalAPI). +var TestLotusToken = os.Getenv("FOREST_JWT") + +func envOr(name, def string) string { + if v := os.Getenv(name); v != "" { + return v + } + return def +} + +// SkipIfNotExternalAPI skips the test if external API access isn't configured. +// Two conditions must hold: SINGULARITY_TEST_EXTERNAL_API opts in, and either +// FOREST_JWT is set or one of the endpoint overrides +// (SINGULARITY_TEST_LOTUS_API / SINGULARITY_TEST_CALIBNET_RPC) points at +// something that doesn't need the gateway's bearer auth. Fork PRs and local +// devs without a token skip cleanly instead of hitting a 401. func SkipIfNotExternalAPI(t *testing.T) { t.Helper() if os.Getenv("SINGULARITY_TEST_EXTERNAL_API") == "" { t.Skip("Skipping external API test. Set SINGULARITY_TEST_EXTERNAL_API=true to run") } + if os.Getenv("FOREST_JWT") == "" && + os.Getenv("SINGULARITY_TEST_LOTUS_API") == "" && + os.Getenv("SINGULARITY_TEST_CALIBNET_RPC") == "" { + t.Skip("Skipping: FOREST_JWT not set and no endpoint override") + } + t.Logf("external API: lotus=%s calibnet=%s token=%t", TestLotusAPI, CalibnetRPC, TestLotusToken != "") } func EscapePath(p string) string { diff --git a/util/util_test.go b/util/util_test.go index 80664e871..b182926d4 100644 --- a/util/util_test.go +++ b/util/util_test.go @@ -25,24 +25,15 @@ func TestNextPowerOfTwo(t *testing.T) { func TestNewLotusClient(t *testing.T) { testutil.SkipIfNotExternalAPI(t) - for _, token := range []string{""} { - t.Run(token, func(t *testing.T) { - client := NewLotusClient(testutil.TestLotusAPI, token) - resp, err := client.Call(context.Background(), "Filecoin.Version") - if token != "" { - require.Error(t, err) - require.ErrorContains(t, err, "401") - return - } - require.NoError(t, err) - require.NotNil(t, resp.Result) - }) - } + client := NewLotusClient(testutil.TestLotusAPI, testutil.TestLotusToken) + resp, err := client.Call(context.Background(), "Filecoin.Version") + require.NoError(t, err) + require.NotNil(t, resp.Result) } func TestGetLotusHeadTime(t *testing.T) { testutil.SkipIfNotExternalAPI(t) - headTime, err := GetLotusHeadTime(context.Background(), testutil.TestLotusAPI, "") + headTime, err := GetLotusHeadTime(context.Background(), testutil.TestLotusAPI, testutil.TestLotusToken) require.NoError(t, err) require.NotZero(t, headTime) }