diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml new file mode 100644 index 0000000..6c95278 --- /dev/null +++ b/.github/workflows/docker-build.yml @@ -0,0 +1,45 @@ +name: Deploy to ECR + +on: + workflow_dispatch: + inputs: + commit_sha: + description: "SHA of the commit to run this workflow on (default is the current commit)" + type: string + +jobs: + build: + name: Build Image + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.commit_sha || github.sha }} + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + + - name: Build, tag, and push image to Amazon ECR + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + ECR_REPOSITORY: wi-backend + IMAGE: google-safebrowsing + run: | + IMAGE_URL="$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE" + + docker build -t $IMAGE_URL . + docker push $IMAGE_URL + + SHORT_SHA=$(git rev-parse --short=7 HEAD) + IMAGE_URL_WITH_SHA="$IMAGE_URL-$SHORT_SHA" + docker tag $IMAGE_URL $IMAGE_URL_WITH_SHA + docker push $IMAGE_URL_WITH_SHA diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..4a29cfc --- /dev/null +++ b/Dockerfile @@ -0,0 +1,21 @@ +# 281284021051.dkr.ecr.us-east-1.amazonaws.com/wi-backend:google-safebrowsing +FROM golang:1.25-alpine AS build + +WORKDIR /src +COPY . . +RUN go build -mod=mod -o /out/sbserver ./cmd/sbserver + +FROM alpine:3.21 + +RUN addgroup -S safebrowsing \ + && adduser -S -G safebrowsing safebrowsing \ + && mkdir -p /var/lib/safebrowsing \ + && chown -R safebrowsing:safebrowsing /var/lib/safebrowsing + +COPY --from=build /out/sbserver /usr/local/bin/sbserver + +USER safebrowsing +EXPOSE 80 +HEALTHCHECK --interval=30s --timeout=5s --start-period=30s --retries=3 \ + CMD wget -q -O /dev/null http://127.0.0.1/health || exit 1 +ENTRYPOINT ["sbserver", "-srvaddr", "0.0.0.0:80"] diff --git a/api.go b/api.go index 921f351..433dfe1 100644 --- a/api.go +++ b/api.go @@ -5,51 +5,84 @@ // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. package safebrowsing import ( - "bytes" "context" + "encoding/base64" "fmt" - "io/ioutil" + "io" "net/http" "net/url" "strings" - - pb "github.com/google/safebrowsing/internal/safebrowsing_proto" + "time" "github.com/golang/protobuf/proto" ) const ( - findHashPath = "/v4/fullHashes:find" - fetchUpdatePath = "/v4/threatListUpdates:fetch" + batchGetPath = "/v5/hashLists:batchGet" + hashSearchPath = "/v5/hashes:search" + maxHashPrefixesPerRequest = 1000 ) -// The api interface specifies wrappers around the Safe Browsing API. +// The api interface specifies wrappers around the Safe Browsing API v5. type api interface { - ListUpdate(ctx context.Context, req *pb.FetchThreatListUpdatesRequest) (*pb.FetchThreatListUpdatesResponse, error) - HashLookup(ctx context.Context, req *pb.FindFullHashesRequest) (*pb.FindFullHashesResponse, error) + ListUpdate(ctx context.Context, names []string, versions map[string][]byte) (*hashListsResponse, error) + HashLookup(ctx context.Context, prefixes []hashPrefix) (*hashSearchResponse, error) } -// netAPI is an api object that talks to the server over HTTP. type netAPI struct { client *http.Client url *url.URL + key string + ua string +} + +type hashListsResponse struct { + HashLists []hashList `json:"hashLists"` +} + +type hashList struct { + Name string `json:"name"` + Version string `json:"version"` + PartialUpdate bool `json:"partialUpdate"` + CompressedRemovals *rice32 `json:"compressedRemovals"` + MinimumWaitDuration string `json:"minimumWaitDuration"` + SHA256Checksum string `json:"sha256Checksum"` + AdditionsFourBytes *rice32 `json:"additionsFourBytes"` + Metadata hashMetadata `json:"metadata"` +} + +type hashMetadata struct { + ThreatTypes []ThreatType `json:"threatTypes"` + HashLength string `json:"hashLength"` +} + +type rice32 struct { + FirstValue uint32 `json:"firstValue"` + RiceParameter int32 `json:"riceParameter"` + EntriesCount int32 `json:"entriesCount"` + EncodedData string `json:"encodedData"` +} + +type hashSearchResponse struct { + FullHashes []fullHash `json:"fullHashes"` + CacheDuration string `json:"cacheDuration"` +} + +type fullHash struct { + FullHash string `json:"fullHash"` + FullHashDetails []fullHashDetail `json:"fullHashDetails"` } -// newNetAPI creates a new netAPI object pointed at the provided root URL. -// For every request, it will use the provided API key. -// If a proxy URL is given, it will be used in place of the default $HTTP_PROXY. -// If the protocol is not specified in root, then this defaults to using HTTPS. -func newNetAPI(root string, key string, proxy string) (*netAPI, error) { +type fullHashDetail struct { + ThreatType ThreatType `json:"threatType"` + Attributes []string `json:"attributes"` +} + +func newNetAPI(root string, key string, proxy string, userAgent string) (*netAPI, error) { if !strings.Contains(root, "://") { root = "https://" + root } @@ -59,59 +92,148 @@ func newNetAPI(root string, key string, proxy string) (*netAPI, error) { } httpClient := &http.Client{} - if proxy != "" { - proxyUrl, err := url.Parse(proxy) + proxyURL, err := url.Parse(proxy) if err != nil { return nil, err } - httpClient = &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxyUrl)}} + httpClient = &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxyURL)}} } + return &netAPI{url: u, client: httpClient, key: key, ua: userAgent}, nil +} + +func (a *netAPI) getProto(ctx context.Context, requestPath string, query url.Values, out proto.Message) error { + u := *a.url + u.Path = requestPath q := u.Query() - q.Set("key", key) - q.Set("alt", "proto") + for k, vals := range query { + for _, v := range vals { + q.Add(k, v) + } + } + q.Set("key", a.key) u.RawQuery = q.Encode() - return &netAPI{url: u, client: httpClient}, nil -} -// doRequests performs a POST to requestPath. It uses the marshaled form of req -// as the request body payload, and automatically unmarshals the response body -// payload as resp. -func (a *netAPI) doRequest(ctx context.Context, requestPath string, req proto.Message, resp proto.Message) error { - p, err := proto.Marshal(req) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) if err != nil { return err } + if a.ua != "" { + req.Header.Set("User-Agent", a.ua) + } + req.Header.Set("Accept", "application/x-protobuf") - u := *a.url // Make a copy of URL - u.Path = requestPath - httpReq, err := http.NewRequest("POST", u.String(), bytes.NewReader(p)) - httpReq.Header.Add("Content-Type", "application/x-protobuf") - httpReq = httpReq.WithContext(ctx) - httpResp, err := a.client.Do(httpReq) + resp, err := a.client.Do(req) if err != nil { return err } - defer httpResp.Body.Close() - if httpResp.StatusCode != 200 { - return fmt.Errorf("safebrowsing: unexpected server response code: %d", httpResp.StatusCode) + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + return fmt.Errorf("safebrowsing: unexpected server response code: %d: %s", resp.StatusCode, strings.TrimSpace(string(body))) } - body, err := ioutil.ReadAll(httpResp.Body) + body, err := io.ReadAll(resp.Body) if err != nil { return err } - return proto.Unmarshal(body, resp) + if err := proto.Unmarshal(body, out); err != nil { + return fmt.Errorf("safebrowsing: decode %s response (%s): %w", requestPath, resp.Header.Get("Content-Type"), err) + } + return nil +} + +func (a *netAPI) ListUpdate(ctx context.Context, names []string, versions map[string][]byte) (*hashListsResponse, error) { + q := url.Values{} + for _, name := range names { + q.Add("names", name) + if version := versions[name]; len(version) > 0 { + q.Add("version", base64.StdEncoding.EncodeToString(version)) + } + } + + var wireResp batchGetHashListsResponse + if err := a.getProto(ctx, batchGetPath, q, &wireResp); err != nil { + return nil, err + } + resp := &hashListsResponse{HashLists: make([]hashList, 0, len(wireResp.HashLists))} + for _, wireList := range wireResp.HashLists { + list := hashList{ + Name: wireList.Name, + Version: base64.StdEncoding.EncodeToString(wireList.Version), + PartialUpdate: wireList.PartialUpdate, + SHA256Checksum: base64.StdEncoding.EncodeToString(wireList.SHA256Checksum), + } + if wait := wireList.MinimumWaitDuration; wait != nil { + list.MinimumWaitDuration = wait.Duration().String() + } + if removals := wireList.CompressedRemovals; removals != nil { + list.CompressedRemovals = rice32FromProto(removals) + } + if additions := wireList.AdditionsFourBytes; additions != nil { + list.AdditionsFourBytes = rice32FromProto(additions) + } + resp.HashLists = append(resp.HashLists, list) + } + return resp, nil } -// ListUpdate issues a FetchThreatListUpdates API call and returns the response. -func (a *netAPI) ListUpdate(ctx context.Context, req *pb.FetchThreatListUpdatesRequest) (*pb.FetchThreatListUpdatesResponse, error) { - resp := new(pb.FetchThreatListUpdatesResponse) - return resp, a.doRequest(ctx, fetchUpdatePath, req, resp) +func (a *netAPI) HashLookup(ctx context.Context, prefixes []hashPrefix) (*hashSearchResponse, error) { + if len(prefixes) > maxHashPrefixesPerRequest { + return nil, fmt.Errorf("safebrowsing: too many hash prefixes: %d", len(prefixes)) + } + q := url.Values{} + for _, prefix := range prefixes { + if len(prefix) != minHashPrefixLength { + return nil, fmt.Errorf("safebrowsing: hash prefix must be exactly %d bytes", minHashPrefixLength) + } + q.Add("hashPrefixes", base64.StdEncoding.EncodeToString([]byte(prefix))) + } + + var wireResp searchHashesResponse + if err := a.getProto(ctx, hashSearchPath, q, &wireResp); err != nil { + return nil, err + } + resp := &hashSearchResponse{FullHashes: make([]fullHash, 0, len(wireResp.FullHashes))} + if duration := wireResp.CacheDuration; duration != nil { + resp.CacheDuration = duration.Duration().String() + } + for _, wireHash := range wireResp.FullHashes { + match := fullHash{ + FullHash: base64.StdEncoding.EncodeToString(wireHash.FullHash), + FullHashDetails: make([]fullHashDetail, 0, len(wireHash.FullHashDetails)), + } + for _, wireDetail := range wireHash.FullHashDetails { + attributes := make([]string, 0, len(wireDetail.Attributes)) + for _, attribute := range wireDetail.Attributes { + attributes = append(attributes, threatAttributeName(attribute)) + } + match.FullHashDetails = append(match.FullHashDetails, fullHashDetail{ + ThreatType: ThreatType(threatTypeName(wireDetail.ThreatType)), + Attributes: attributes, + }) + } + resp.FullHashes = append(resp.FullHashes, match) + } + return resp, nil } -// HashLookup issues a FindFullHashes API call and returns the response. -func (a *netAPI) HashLookup(ctx context.Context, req *pb.FindFullHashesRequest) (*pb.FindFullHashesResponse, error) { - resp := new(pb.FindFullHashesResponse) - return resp, a.doRequest(ctx, findHashPath, req, resp) +func rice32FromProto(value *riceDeltaEncoded32Bit) *rice32 { + return &rice32{ + FirstValue: value.FirstValue, + RiceParameter: value.RiceParameter, + EntriesCount: value.EntriesCount, + EncodedData: base64.StdEncoding.EncodeToString(value.EncodedData), + } +} + +func parseDuration(s string) time.Duration { + if s == "" { + return 0 + } + d, err := time.ParseDuration(s) + if err != nil { + return 0 + } + return d } diff --git a/api_test.go b/api_test.go deleted file mode 100644 index 03c09b2..0000000 --- a/api_test.go +++ /dev/null @@ -1,141 +0,0 @@ -// Copyright 2016 Google Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package safebrowsing - -import ( - "context" - "io/ioutil" - "net/http" - "net/http/httptest" - "testing" - - pb "github.com/google/safebrowsing/internal/safebrowsing_proto" - - "github.com/golang/protobuf/proto" -) - -type mockAPI struct { - listUpdate func(context.Context, *pb.FetchThreatListUpdatesRequest) (*pb.FetchThreatListUpdatesResponse, error) - hashLookup func(context.Context, *pb.FindFullHashesRequest) (*pb.FindFullHashesResponse, error) -} - -func (m *mockAPI) ListUpdate(ctx context.Context, req *pb.FetchThreatListUpdatesRequest) (*pb.FetchThreatListUpdatesResponse, error) { - return m.listUpdate(ctx, req) -} - -func (m *mockAPI) HashLookup(ctx context.Context, req *pb.FindFullHashesRequest) (*pb.FindFullHashesResponse, error) { - return m.hashLookup(ctx, req) -} - -func TestNetAPI(t *testing.T) { - var gotReq, wantReq proto.Message - var gotResp, wantResp proto.Message - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - var p []byte - var err error - if p, err = ioutil.ReadAll(r.Body); err != nil { - t.Fatalf("unexpected ioutil.ReadAll error: %v", err) - } - if err := proto.Unmarshal(p, gotReq); err != nil { - t.Fatalf("unexpected proto.Unmarshal error: %v", err) - } - if p, err = proto.Marshal(wantResp); err != nil { - t.Fatalf("unexpected proto.Marshal error: %v", err) - } - if _, err := w.Write(p); err != nil { - t.Fatalf("unexpected ResponseWriter.Write error: %v", err) - } - })) - defer ts.Close() - - api, err := newNetAPI(ts.URL, "fizzbuzz", "") - if err != nil { - t.Errorf("unexpected newNetAPI error: %v", err) - } - - // Test that ListUpdate marshal/unmarshal works. - wantReq = &pb.FetchThreatListUpdatesRequest{ListUpdateRequests: []*pb.FetchThreatListUpdatesRequest_ListUpdateRequest{ - {ThreatType: 0, PlatformType: 1, ThreatEntryType: 2, State: []byte("meow")}, - {ThreatType: 1, PlatformType: 2, ThreatEntryType: 3, State: []byte("rawr")}, - {ThreatType: 3, PlatformType: 4, ThreatEntryType: 5, - Constraints: &pb.FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints{ - SupportedCompressions: []pb.CompressionType{1, 2, 3}}}, - }} - wantResp = &pb.FetchThreatListUpdatesResponse{ListUpdateResponses: []*pb.FetchThreatListUpdatesResponse_ListUpdateResponse{ - {ThreatType: 0, PlatformType: 1, ThreatEntryType: 2, NewClientState: []byte("meow")}, - {ThreatType: 1, PlatformType: 2, ThreatEntryType: 3, ResponseType: 1}, - {ThreatType: 2, PlatformType: 3, ThreatEntryType: 4, Checksum: &pb.Checksum{Sha256: []byte("abcd")}}, - {ThreatType: 3, PlatformType: 4, ThreatEntryType: 5, Removals: []*pb.ThreatEntrySet{{ - CompressionType: 1, RawIndices: &pb.RawIndices{Indices: []int32{1, 2, 3}}, - }}}, - }} - gotReq = &pb.FetchThreatListUpdatesRequest{} - resp1, err := api.ListUpdate(context.Background(), wantReq.(*pb.FetchThreatListUpdatesRequest)) - gotResp = resp1 - if err != nil { - t.Errorf("unexpected ListUpdate error: %v", err) - } - if !proto.Equal(gotReq, wantReq) { - t.Errorf("mismatching ListUpdate requests:\ngot %+v\nwant %+v", gotReq, wantReq) - } - if !proto.Equal(gotResp, wantResp) { - t.Errorf("mismatching ListUpdate responses:\ngot %+v\nwant %+v", gotResp, wantResp) - } - - // Test that HashLookup marshal/unmarshal works. - wantReq = &pb.FindFullHashesRequest{ThreatInfo: &pb.ThreatInfo{ - ThreatEntries: []*pb.ThreatEntry{{Hash: []byte("aaaa")}, {Hash: []byte("bbbbb")}, {Hash: []byte("cccccc")}}, - ThreatTypes: []pb.ThreatType{1, 2, 3}, - PlatformTypes: []pb.PlatformType{4, 5, 6}, - ThreatEntryTypes: []pb.ThreatEntryType{7, 8, 9}, - }} - wantResp = &pb.FindFullHashesResponse{Matches: []*pb.ThreatMatch{ - {ThreatType: 0, PlatformType: 1, ThreatEntryType: 2, Threat: &pb.ThreatEntry{Hash: []byte("abcd")}}, - {ThreatType: 1, PlatformType: 2, ThreatEntryType: 3, Threat: &pb.ThreatEntry{Hash: []byte("efgh")}}, - {ThreatType: 2, PlatformType: 3, ThreatEntryType: 4, Threat: &pb.ThreatEntry{Hash: []byte("ijkl")}}, - }} - gotReq = &pb.FindFullHashesRequest{} - resp2, err := api.HashLookup(context.Background(), wantReq.(*pb.FindFullHashesRequest)) - gotResp = resp2 - if err != nil { - t.Errorf("unexpected HashLookup error: %v", err) - } - if !proto.Equal(gotReq, wantReq) { - t.Errorf("mismatching HashLookup requests:\ngot %+v\nwant %+v", gotReq, wantReq) - } - if !proto.Equal(gotResp, wantResp) { - t.Errorf("mismatching HashLookup responses:\ngot %+v\nwant %+v", gotResp, wantResp) - } - - // Test canceled Context returns an error. - wantReq = &pb.FindFullHashesRequest{ThreatInfo: &pb.ThreatInfo{ - ThreatEntries: []*pb.ThreatEntry{{Hash: []byte("aaaa")}, {Hash: []byte("bbbbb")}, {Hash: []byte("cccccc")}}, - ThreatTypes: []pb.ThreatType{1, 2, 3}, - PlatformTypes: []pb.PlatformType{4, 5, 6}, - ThreatEntryTypes: []pb.ThreatEntryType{7, 8, 9}, - }} - wantResp = &pb.FindFullHashesResponse{Matches: []*pb.ThreatMatch{ - {ThreatType: 0, PlatformType: 1, ThreatEntryType: 2, Threat: &pb.ThreatEntry{Hash: []byte("abcd")}}, - {ThreatType: 1, PlatformType: 2, ThreatEntryType: 3, Threat: &pb.ThreatEntry{Hash: []byte("efgh")}}, - {ThreatType: 2, PlatformType: 3, ThreatEntryType: 4, Threat: &pb.ThreatEntry{Hash: []byte("ijkl")}}, - }} - gotReq = &pb.FindFullHashesRequest{} - ctx, cancel := context.WithCancel(context.Background()) - cancel() - _, err = api.HashLookup(ctx, wantReq.(*pb.FindFullHashesRequest)) - if err == nil { - t.Errorf("unexpected HashLookup success, wanted HTTP request canceled") - } -} diff --git a/api_wire.go b/api_wire.go new file mode 100644 index 0000000..32c633e --- /dev/null +++ b/api_wire.go @@ -0,0 +1,106 @@ +package safebrowsing + +import ( + "time" + + "github.com/golang/protobuf/proto" +) + +type batchGetHashListsResponse struct { + HashLists []*wireHashList `protobuf:"bytes,1,rep,name=hash_lists,json=hashLists,proto3"` +} + +func (m *batchGetHashListsResponse) Reset() { *m = batchGetHashListsResponse{} } +func (m *batchGetHashListsResponse) String() string { return proto.CompactTextString(m) } +func (*batchGetHashListsResponse) ProtoMessage() {} + +type wireHashList struct { + Name string `protobuf:"bytes,1,opt,name=name,proto3"` + Version []byte `protobuf:"bytes,2,opt,name=version,proto3"` + PartialUpdate bool `protobuf:"varint,3,opt,name=partial_update,json=partialUpdate,proto3"` + AdditionsFourBytes *riceDeltaEncoded32Bit `protobuf:"bytes,4,opt,name=additions_four_bytes,json=additionsFourBytes,proto3"` + CompressedRemovals *riceDeltaEncoded32Bit `protobuf:"bytes,5,opt,name=compressed_removals,json=compressedRemovals,proto3"` + MinimumWaitDuration *wireDuration `protobuf:"bytes,6,opt,name=minimum_wait_duration,json=minimumWaitDuration,proto3"` + SHA256Checksum []byte `protobuf:"bytes,7,opt,name=sha256_checksum,json=sha256Checksum,proto3"` +} + +func (m *wireHashList) Reset() { *m = wireHashList{} } +func (m *wireHashList) String() string { return proto.CompactTextString(m) } +func (*wireHashList) ProtoMessage() {} + +type riceDeltaEncoded32Bit struct { + FirstValue uint32 `protobuf:"varint,1,opt,name=first_value,json=firstValue,proto3"` + RiceParameter int32 `protobuf:"varint,2,opt,name=rice_parameter,json=riceParameter,proto3"` + EntriesCount int32 `protobuf:"varint,3,opt,name=entries_count,json=entriesCount,proto3"` + EncodedData []byte `protobuf:"bytes,4,opt,name=encoded_data,json=encodedData,proto3"` +} + +func (m *riceDeltaEncoded32Bit) Reset() { *m = riceDeltaEncoded32Bit{} } +func (m *riceDeltaEncoded32Bit) String() string { return proto.CompactTextString(m) } +func (*riceDeltaEncoded32Bit) ProtoMessage() {} + +type wireDuration struct { + Seconds int64 `protobuf:"varint,1,opt,name=seconds,proto3"` + Nanos int32 `protobuf:"varint,2,opt,name=nanos,proto3"` +} + +func (m *wireDuration) Reset() { *m = wireDuration{} } +func (m *wireDuration) String() string { return proto.CompactTextString(m) } +func (*wireDuration) ProtoMessage() {} + +func (m *wireDuration) Duration() time.Duration { + return time.Duration(m.Seconds)*time.Second + time.Duration(m.Nanos) +} + +type searchHashesResponse struct { + FullHashes []*wireFullHash `protobuf:"bytes,1,rep,name=full_hashes,json=fullHashes,proto3"` + CacheDuration *wireDuration `protobuf:"bytes,2,opt,name=cache_duration,json=cacheDuration,proto3"` +} + +func (m *searchHashesResponse) Reset() { *m = searchHashesResponse{} } +func (m *searchHashesResponse) String() string { return proto.CompactTextString(m) } +func (*searchHashesResponse) ProtoMessage() {} + +type wireFullHash struct { + FullHash []byte `protobuf:"bytes,1,opt,name=full_hash,json=fullHash,proto3"` + FullHashDetails []*wireFullHashDetail `protobuf:"bytes,2,rep,name=full_hash_details,json=fullHashDetails,proto3"` +} + +func (m *wireFullHash) Reset() { *m = wireFullHash{} } +func (m *wireFullHash) String() string { return proto.CompactTextString(m) } +func (*wireFullHash) ProtoMessage() {} + +type wireFullHashDetail struct { + ThreatType int32 `protobuf:"varint,1,opt,name=threat_type,json=threatType,proto3"` + Attributes []int32 `protobuf:"varint,2,rep,packed,name=attributes,proto3"` +} + +func (m *wireFullHashDetail) Reset() { *m = wireFullHashDetail{} } +func (m *wireFullHashDetail) String() string { return proto.CompactTextString(m) } +func (*wireFullHashDetail) ProtoMessage() {} + +func threatTypeName(value int32) string { + switch value { + case 1: + return string(ThreatTypeMalware) + case 2: + return string(ThreatTypeSocialEngineering) + case 3: + return string(ThreatTypeUnwantedSoftware) + case 4: + return string(ThreatTypePotentiallyHarmfulApplication) + default: + return string(ThreatTypeUnspecified) + } +} + +func threatAttributeName(value int32) string { + switch value { + case 1: + return "CANARY" + case 2: + return "FRAME_ONLY" + default: + return "THREAT_ATTRIBUTE_UNSPECIFIED" + } +} diff --git a/cache.go b/cache.go index f5722dd..508ba7c 100644 --- a/cache.go +++ b/cache.go @@ -5,109 +5,64 @@ // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. package safebrowsing import ( "sync" "time" - - pb "github.com/google/safebrowsing/internal/safebrowsing_proto" ) type cacheResult int const ( - // positiveCacheHit indicates that the given hash matched an entry in the cache. - // The caller must consider the match a threat and not contact the server. positiveCacheHit cacheResult = iota - - // negativeCacheHit indicates that the given hash did not match any entries - // in the cache but its prefix matches the negative cache. The caller must - // consider the given hash to be safe and not contact the server. negativeCacheHit - - // cacheMiss indicates that the given hash did not match any entry - // in the cache. The caller should make a follow-up query to the server. cacheMiss ) -// cache caches results from API calls to FindFullHashesRequest to reduce -// network calls for recently requested items. Since the global blacklist is -// constantly changing, the Safe Browsing API defines TTLs for how long entries -// can stay alive in the cache. type cache struct { sync.RWMutex - - // pttls maps full hashes and a ThreatDescriptor to a positive time-to-live. - // For a given full hash, the known threats are all ThreatDescriptors that - // map to valid TTLs (i.e. in the future). - pttls map[hashPrefix]map[ThreatDescriptor]time.Time - - // nttls maps partial hashes to a negative time-to-live. - // If this is still valid (i.e. in the future), then this indicates that - // there are *no* threats under the given partial hash, unless there exist - // ThreatDescriptors with a valid positive TTL for that hash. + pttls map[hashPrefix]map[ThreatType]time.Time nttls map[hashPrefix]time.Time - - now func() time.Time + now func() time.Time } -// Update updates the cache according to the request that was made to the server -// and the response given back. -func (c *cache) Update(req *pb.FindFullHashesRequest, resp *pb.FindFullHashesResponse) { +func (c *cache) Update(prefixes []hashPrefix, resp *hashSearchResponse) { c.Lock() defer c.Unlock() - now := c.now() if c.pttls == nil { - c.pttls = make(map[hashPrefix]map[ThreatDescriptor]time.Time) + c.pttls = make(map[hashPrefix]map[ThreatType]time.Time) c.nttls = make(map[hashPrefix]time.Time) } - // Insert each threat match into the cache by full hash. - for _, tm := range resp.GetMatches() { - fullHash := hashPrefix(tm.GetThreat().Hash) + expiration := c.now().Add(parseDuration(resp.CacheDuration)) + for _, prefix := range prefixes { + c.nttls[prefix] = expiration + } + + for _, match := range resp.FullHashes { + fullHashBytes, err := decodeBase64(match.FullHash) + if err != nil { + continue + } + fullHash := hashPrefix(fullHashBytes) if !fullHash.IsFull() { continue } if c.pttls[fullHash] == nil { - c.pttls[fullHash] = make(map[ThreatDescriptor]time.Time) - } - var dur time.Duration - if tmCacheDur := tm.GetCacheDuration(); tmCacheDur != nil { - dur = time.Duration(tm.GetCacheDuration().Seconds) * time.Second - } else { - dur = 0 + c.pttls[fullHash] = make(map[ThreatType]time.Time) } - td := ThreatDescriptor{ - ThreatType: ThreatType(tm.ThreatType), - PlatformType: PlatformType(tm.PlatformType), - ThreatEntryType: ThreatEntryType(tm.ThreatEntryType), - } - c.pttls[fullHash][td] = now.Add(dur) - } - - // Insert negative TTLs for partial hashes. - if resp.GetNegativeCacheDuration() != nil { - dur := time.Duration(resp.GetNegativeCacheDuration().Seconds) * time.Second - nttl := now.Add(dur) - for _, te := range req.GetThreatInfo().GetThreatEntries() { - partialHash := hashPrefix(te.Hash) - c.nttls[partialHash] = nttl + for _, detail := range match.FullHashDetails { + if enforceableThreatDetail(detail) { + c.pttls[fullHash][detail.ThreatType] = expiration + } } } } -// Lookup looks up a full hash and returns a set of ThreatDescriptors and the -// validity of the result. -func (c *cache) Lookup(hash hashPrefix) (map[ThreatDescriptor]bool, cacheResult) { +func (c *cache) Lookup(hash hashPrefix) (map[ThreatType]bool, cacheResult) { if !hash.IsFull() { panic("hash is not full") } @@ -116,66 +71,40 @@ func (c *cache) Lookup(hash hashPrefix) (map[ThreatDescriptor]bool, cacheResult) defer c.Unlock() now := c.now() - // Check all entries to see if there *is* a threat. - threats := make(map[ThreatDescriptor]bool) - threatTTLs := c.pttls[hash] - for td, pttl := range threatTTLs { + threats := make(map[ThreatType]bool) + for threatType, pttl := range c.pttls[hash] { if pttl.After(now) { - threats[td] = true + threats[threatType] = true } else { - // The PTTL has expired, we should ask the server what's going on. return nil, cacheMiss } } if len(threats) > 0 { - // So long as there are valid threats, we report them. The positive TTL - // takes precedence over the negative TTL at the partial hash level. return threats, positiveCacheHit } - // Check the negative TTLs to see if there are *no* threats. - for i := minHashPrefixLength; i <= maxHashPrefixLength; i++ { - if nttl, ok := c.nttls[hash[:i]]; ok { - if nttl.After(now) { - return nil, negativeCacheHit - } - } + prefix := hash[:minHashPrefixLength] + if nttl, ok := c.nttls[prefix]; ok && nttl.After(now) { + return nil, negativeCacheHit } - - // The cache has no information; it is a *possible* threat. return nil, cacheMiss } -// Purge purges all expired entries from the cache. func (c *cache) Purge() { c.Lock() defer c.Unlock() now := c.now() - // Nuke all threat entries based on their positive TTL. for fullHash, threatTTLs := range c.pttls { - for td, pttl := range threatTTLs { + for threatType, pttl := range threatTTLs { if now.After(pttl) { - del := true - for i := minHashPrefixLength; i <= maxHashPrefixLength; i++ { - if nttl, ok := c.nttls[fullHash[:i]]; ok { - if nttl.After(pttl) { - del = false - break - } - } - } - if del { - delete(threatTTLs, td) - } + delete(threatTTLs, threatType) } } if len(threatTTLs) == 0 { delete(c.pttls, fullHash) } } - - // Nuke all partial hashes based on their negative TTL. for partialHash, nttl := range c.nttls { if now.After(nttl) { delete(c.nttls, partialHash) diff --git a/cache_test.go b/cache_test.go deleted file mode 100644 index 2cfeff4..0000000 --- a/cache_test.go +++ /dev/null @@ -1,247 +0,0 @@ -// Copyright 2016 Google Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package safebrowsing - -import ( - "reflect" - "testing" - "time" - - dpb "github.com/golang/protobuf/ptypes/duration" - pb "github.com/google/safebrowsing/internal/safebrowsing_proto" -) - -func TestCacheLookup(t *testing.T) { - now := time.Unix(1451436338, 951473000) - mockNow := func() time.Time { return now } - - type cacheLookup struct { - h hashPrefix - tds map[ThreatDescriptor]bool - r cacheResult - } - vectors := []struct { - gotCache *cache // The cache to apply the Purge and Lookup on - wantCache *cache // The cache expected after Purge - lookups []cacheLookup - }{{ - gotCache: &cache{ - pttls: map[hashPrefix]map[ThreatDescriptor]time.Time{ - "AAAABBBBBBBBBBBBBBBBBBBBBBBBBBBB": { - {1, 2, 3}: now.Add(DefaultUpdatePeriod), - }, - "ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ": { - {2, 2, 2}: now.Add(-time.Minute), - {1, 1, 1}: now.Add(-DefaultUpdatePeriod), - }, - }, - nttls: map[hashPrefix]time.Time{ - "AAAA": now.Add(DefaultUpdatePeriod), - "BBBB": now.Add(-time.Minute), - }, - now: mockNow, - }, - wantCache: &cache{ - pttls: map[hashPrefix]map[ThreatDescriptor]time.Time{ - "AAAABBBBBBBBBBBBBBBBBBBBBBBBBBBB": { - {1, 2, 3}: now.Add(DefaultUpdatePeriod), - }, - }, - nttls: map[hashPrefix]time.Time{ - "AAAA": now.Add(DefaultUpdatePeriod), - }, - now: mockNow, - }, - lookups: []cacheLookup{{ - h: "AAAABBBBBBBBBBBBBBBBBBBBBBBBBBBB", - tds: map[ThreatDescriptor]bool{{1, 2, 3}: true}, - r: positiveCacheHit, - }, { - h: "AAAACDCDCDCDCDCDCDCDCDCDCDCDCDCD", - tds: nil, - r: negativeCacheHit, - }, { - h: "ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ", - tds: nil, - r: cacheMiss, - }}, - }, { - gotCache: &cache{ - pttls: map[hashPrefix]map[ThreatDescriptor]time.Time{ - "AAAABBBBBBBBBBBBBBBBBBBBBBBBBBBB": { - {1, 2, 3}: now.Add(DefaultUpdatePeriod), - {1, 1, 1}: now.Add(-DefaultUpdatePeriod), - }, - "ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ": { - {2, 2, 2}: now.Add(-time.Minute), - {1, 1, 1}: now.Add(-DefaultUpdatePeriod), - }, - }, - nttls: map[hashPrefix]time.Time{ - "AAAA": now.Add(DefaultUpdatePeriod * 2), - "BBBB": now.Add(-time.Minute), - }, - now: mockNow, - }, - wantCache: &cache{ - pttls: map[hashPrefix]map[ThreatDescriptor]time.Time{ - "AAAABBBBBBBBBBBBBBBBBBBBBBBBBBBB": { - {1, 2, 3}: now.Add(DefaultUpdatePeriod), - {1, 1, 1}: now.Add(-DefaultUpdatePeriod), - }, - }, - nttls: map[hashPrefix]time.Time{ - "AAAA": now.Add(DefaultUpdatePeriod * 2), - }, - now: mockNow, - }, - lookups: []cacheLookup{{ - h: "AAAABBBBBBBBBBBBBBBBBBBBBBBBBBBB", - tds: nil, - r: cacheMiss, - }, { - h: "AAAACDCDCDCDCDCDCDCDCDCDCDCDCDCD", - tds: nil, - r: negativeCacheHit, - }, { - h: "ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ", - tds: nil, - r: cacheMiss, - }}, - }, { - gotCache: &cache{now: mockNow}, - wantCache: &cache{now: mockNow}, - lookups: []cacheLookup{{ - h: "AAAABBBBBBBBBBBBBBBBBBBBBBBBBBBB", - tds: nil, - r: cacheMiss, - }, { - h: "AAAACDCDCDCDCDCDCDCDCDCDCDCDCDCD", - tds: nil, - r: cacheMiss, - }}, - }} - - for i, v := range vectors { - for j, l := range v.lookups { - gotTDs, gotR := v.gotCache.Lookup(l.h) - if !reflect.DeepEqual(gotTDs, l.tds) { - t.Errorf("test %d, lookup %d, threats mismatch:\ngot %+v\nwant %+v", i, j, gotTDs, l.tds) - } - if gotR != l.r { - t.Errorf("test %d, lookup %d, result mismatch: got %d, want %d", i, j, gotR, l.r) - - } - } - v.gotCache.Purge() - if !reflect.DeepEqual(v.wantCache.pttls, v.gotCache.pttls) { - t.Errorf("purge test %d, mismatching cache contents: PTTLS\ngot %+v\nwant %+v", i, v.gotCache.pttls, v.wantCache.pttls) - } - if !reflect.DeepEqual(v.wantCache.nttls, v.gotCache.nttls) { - t.Errorf("purge test %d, mismatching cache contents: NTTLS\ngot %+v\nwant %+v", i, v.gotCache.nttls, v.wantCache.nttls) - } - for j, l := range v.lookups { - gotTDs, gotR := v.gotCache.Lookup(l.h) - if !reflect.DeepEqual(gotTDs, l.tds) { - t.Errorf("purge test %d, lookup %d, threats mismatch:\ngot %+v\nwant %+v", i, j, gotTDs, l.tds) - } - if gotR != l.r { - t.Errorf("purge test %d, lookup %d, result mismatch: got %d, want %d", i, j, gotR, l.r) - } - } - } -} - -func TestCacheUpdate(t *testing.T) { - now := time.Unix(1451436338, 951473000) - mockNow := func() time.Time { return now } - - vectors := []struct { - req *pb.FindFullHashesRequest - resp *pb.FindFullHashesResponse - gotCache *cache - wantCache *cache - }{{ - req: &pb.FindFullHashesRequest{}, - resp: &pb.FindFullHashesResponse{}, - gotCache: &cache{ - now: mockNow, - }, - wantCache: &cache{pttls: map[hashPrefix]map[ThreatDescriptor]time.Time{}, - nttls: map[hashPrefix]time.Time{}, - now: mockNow, - }, - }, { - req: &pb.FindFullHashesRequest{ - ThreatInfo: &pb.ThreatInfo{ - ThreatTypes: []pb.ThreatType{0, 1, 2}, - PlatformTypes: []pb.PlatformType{1, 2, 3}, - ThreatEntryTypes: []pb.ThreatEntryType{2, 3, 4}, - ThreatEntries: []*pb.ThreatEntry{{Hash: []byte("aaaa")}}, - }}, - resp: &pb.FindFullHashesResponse{ - Matches: []*pb.ThreatMatch{{ - ThreatType: 0, - PlatformType: 1, - ThreatEntryType: 2, - Threat: &pb.ThreatEntry{Hash: []byte("aaaabbbbccccddddeeeeffffgggghhhh")}, - CacheDuration: &dpb.Duration{Seconds: 1000, Nanos: 0}, - }, { - ThreatType: 1, - PlatformType: 2, - ThreatEntryType: 3, - Threat: &pb.ThreatEntry{Hash: []byte("aaaaaaaaccccddddeeeeffffgggghhhh")}, - CacheDuration: &dpb.Duration{Seconds: 1000, Nanos: 0}, - }, { - ThreatType: 2, - PlatformType: 3, - ThreatEntryType: 4, - Threat: &pb.ThreatEntry{Hash: []byte("aaaaccccccccddddeeeeffffgggghhhh")}, - CacheDuration: &dpb.Duration{Seconds: 1000, Nanos: 0}, - }}, - NegativeCacheDuration: &dpb.Duration{Seconds: 1000, Nanos: 0}, - }, - gotCache: &cache{ - now: mockNow, - }, - wantCache: &cache{ - pttls: map[hashPrefix]map[ThreatDescriptor]time.Time{ - "aaaabbbbccccddddeeeeffffgggghhhh": { - {0, 1, 2}: now.Add(1000 * time.Second), - }, - "aaaaaaaaccccddddeeeeffffgggghhhh": { - {1, 2, 3}: now.Add(1000 * time.Second), - }, - "aaaaccccccccddddeeeeffffgggghhhh": { - {2, 3, 4}: now.Add(1000 * time.Second), - }, - }, - nttls: map[hashPrefix]time.Time{ - "aaaa": now.Add(1000 * time.Second), - }, - now: mockNow, - }, - }} - - for i, v := range vectors { - v.gotCache.Update(v.req, v.resp) - if !reflect.DeepEqual(v.wantCache.pttls, v.gotCache.pttls) { - t.Errorf("test %d, mismatching cache contents: PTTLS\ngot %+v\nwant %+v", i, v.gotCache.pttls, v.wantCache.pttls) - } - if !reflect.DeepEqual(v.wantCache.nttls, v.gotCache.nttls) { - t.Errorf("test %d, mismatching cache contents: NTTLS\ngot %+v\nwant %+v", i, v.gotCache.nttls, v.wantCache.nttls) - } - } -} diff --git a/cmd/sbserver/main.go b/cmd/sbserver/main.go index 6c8c9fd..d7f32dd 100644 --- a/cmd/sbserver/main.go +++ b/cmd/sbserver/main.go @@ -5,439 +5,195 @@ // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// Command sbserver is an application for serving URL lookups via a simple API. -// -// In order to abstract away the complexities of the Safe Browsing API v4, the -// sbserver application can be used to serve a subset of API v4 over HTTP. -// This subset is intentionally small so that it would be easy to implement by -// a client. It is intended for sbserver to either be running locally on a -// client's machine or within the same local network. That way it can handle -// most local API calls before resorting to making an API call to the actual -// Safe Browsing API over the internet. -// -// Usage of sbserver looks something like this: -// _________________ -// | | -// | Safe Browsing | -// | API v4 servers | -// |_________________| -// | -// ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ -// The Internet -// ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ -// | -// _______V_______ -// | | -// | SBServer | -// +------| Application |------+ -// | |_______________| | -// | | | -// _____V_____ _____V_____ _____V_____ -// | | | | | | -// | Client1 | | Client2 | | Client3 | -// |___________| |___________| |___________| -// -// In theory, each client could directly use the Go SafeBrowser implementation, -// but there are situations where that is not desirable. For example, the client -// may not be using the language Go, or there may be multiple clients in the -// same machine or local network that would like to share a local database -// and cache. The sbserver was designed to address these issues. -// -// The sbserver application is technically a proxy since it is itself actually -// an API v4 client. It connects the Safe Browsing API servers using an API key -// and maintains a local database and cache. However, it is also a server since -// it re-serves a subset of the API v4 endpoints. These endpoints are minimal -// in that they do not require each client to maintain state between calls. -// -// The assumption is that communication between SBServer and Client1, Client2, -// and Client3 is inexpensive, since they are within the same machine or same -// local network. Thus, the sbserver can efficiently satisfy some requests -// without talking to the global Safe Browsing servers since it has a -// potentially larger cache. Furthermore, it can multiplex multiple requests to -// the Safe Browsing servers on fewer TCP connections, reducing the cost for -// comparatively more expensive internet transfers. -// -// By default, the sbserver listens on localhost:8080 and serves the following -// API endpoints: -// /v4/threatMatches:find -// /v4/threatLists -// /status -// /r -// -// -// Endpoint: /v4/threatMatches:find -// -// This is a lightweight implementation of the API v4 threatMatches endpoint. -// Essentially, it takes in a list of URLs, and returns a list of threat matches -// for those URLs. Unlike the Safe Browsing API, it does not require an API key. -// -// Example usage: -// # Send request to server: -// $ curl \ -// -H "Content-Type: application/json" \ -// -X POST -d '{ -// "threatInfo": { -// "threatTypes": ["UNWANTED_SOFTWARE", "MALWARE"], -// "platformTypes": ["ANY_PLATFORM"], -// "threatEntryTypes": ["URL"], -// "threatEntries": [ -// {"url": "google.com"}, -// {"url": "bad1url.org"}, -// {"url": "bad2url.org"} -// ] -// } -// }' \ -// localhost:8080/v4/threatMatches:find -// -// # Receive response from server: -// { -// "matches": [{ -// "threat": {"url": "bad1url.org"}, -// "platformType": "ANY_PLATFORM", -// "threatType": "UNWANTED_SOFTWARE", -// "threatEntryType": "URL" -// }, { -// "threat": {"url": "bad2url.org"}, -// "platformType": "ANY_PLATFORM", -// "threatType": "UNWANTED_SOFTWARE", -// "threatEntryType": "URL" -// }, { -// "threat": {"url": "bad2url.org"}, -// "platformType": "ANY_PLATFORM", -// "threatType": "MALWARE", -// "threatEntryType": "URL" -// }] -// } -// -// -// Endpoint: /v4/threatLists -// -// The endpoint returns a list of the threat lists that the sbserver is -// currently subscribed to. The threats returned by the earlier threatMatches -// API call may only be one of these types. -// -// Example usage: -// # Send request to server: -// $ curl -X GET localhost:8080/v4/threatLists -// -// # Receive response from server: -// { -// "threatLists": [{ -// "threatType": "MALWARE" -// "platformType": "ANY_PLATFORM", -// "threatEntryType": "URL", -// }, { -// "threatType": "SOCIAL_ENGINEERING", -// "platformType": "ANY_PLATFORM" -// "threatEntryType": "URL", -// }, { -// "threatType": "UNWANTED_SOFTWARE" -// "platformType": "ANY_PLATFORM", -// "threatEntryType": "URL", -// }] -// } -// -// -// Endpoint: /status -// -// The status endpoint allows a client to obtain some statistical information -// regarding the health of sbserver. It can be used to determine how many -// requests were satisfied locally by sbserver alone and how many requests -// were forwarded to the Safe Browsing API servers. -// -// Example usage: -// $ curl localhost:8080/status -// { -// "Stats" : { -// "QueriesByDatabase" : 132, -// "QueriesByCache" : 31, -// "QueriesByAPI" : 6, -// "QueriesFail" : 0, -// }, -// "Error" : "" -// } -// -// -// Endpoint: /r -// -// The redirector endpoint allows a client to pass in a query URL. -// If the URL is safe, the client is automatically redirected to the target. -// If the URL is unsafe, then an interstitial warning page is shown instead. -// -// Example usage: -// $ curl -i localhost:8080/r?url=http://google.com -// HTTP/1.1 302 Found -// Location: http://google.com -// -// $ curl -i localhost:8080/r?url=http://bad1url.org -// HTTP/1.1 200 OK -// Date: Wed, 13 Apr 2016 21:29:33 GMT -// Content-Length: 1783 -// Content-Type: text/html; charset=utf-8 -// -// -// ... package main import ( - "bytes" "encoding/json" - "errors" "flag" "fmt" "html/template" "io/ioutil" - "mime" "net/http" "net/url" "os" + "time" "github.com/google/safebrowsing" - pb "github.com/google/safebrowsing/internal/safebrowsing_proto" - - "github.com/golang/protobuf/jsonpb" - "github.com/golang/protobuf/proto" _ "github.com/google/safebrowsing/cmd/sbserver/statik" "github.com/rakyll/statik/fs" ) const ( - statusPath = "/status" - findThreatPath = "/v4/threatMatches:find" - getThreatListsPath = "/v4/threatLists" - redirectPath = "/r" -) - -const ( - mimeJSON = "application/json" - mimeProto = "application/x-protobuf" + healthPath = "/health" + statusPath = "/status" + searchURLsPath = "/v5/urls:search" + getHashListsPath = "/v5/hashLists" + redirectPath = "/r" + mimeJSON = "application/json" ) var ( apiKeyFlag = flag.String("apikey", "", "specify your Safe Browsing API key") srvAddrFlag = flag.String("srvaddr", "localhost:8080", "TCP network address the HTTP server should use") proxyFlag = flag.String("proxy", "", "proxy to use to connect to the HTTP server") - databaseFlag = flag.String("db", "", "path to the Safe Browsing database.") + databaseFlag = flag.String("db", "/var/lib/safebrowsing/safebrowsing.gob", "path to the Safe Browsing database") ) var threatTemplate = map[safebrowsing.ThreatType]string{ - safebrowsing.ThreatType_Malware: "/malware.tmpl", - safebrowsing.ThreatType_PotentiallyHarmfulApplication: "/malware.tmpl", - safebrowsing.ThreatType_UnwantedSoftware: "/unwanted.tmpl", - safebrowsing.ThreatType_SocialEngineering: "/social_engineering.tmpl", + safebrowsing.ThreatTypeMalware: "/malware.tmpl", + safebrowsing.ThreatTypePotentiallyHarmfulApplication: "/malware.tmpl", + safebrowsing.ThreatTypeUnwantedSoftware: "/unwanted.tmpl", + safebrowsing.ThreatTypeSocialEngineering: "/social_engineering.tmpl", } -const usage = `sbserver: starts a Safe Browsing API proxy server. - -In order to abstract away the complexities of the Safe Browsing API v4, the -sbserver application can be used to serve a subset of the v4 API. -This subset is intentionally small so that it would be easy to implement by -a client. It is intended for sbserver to either be running locally on a -client's machine or within the same local network so that it can handle most -local API calls before resorting to making an API call to the actual -Safe Browsing API over the internet. +type urlSearchRequest struct { + URL string `json:"url"` + URLs []string `json:"urls"` +} -Usage: %s -apikey=$APIKEY +type urlSearchResponse struct { + Matches []urlMatch `json:"matches,omitempty"` +} -` +type urlMatch struct { + URL string `json:"url"` + ThreatType string `json:"threatType"` + HashList string `json:"hashList"` + Pattern string `json:"pattern,omitempty"` +} -// unmarshal reads pbResp from req. The mime will either be JSON or ProtoBuf. -func unmarshal(req *http.Request, pbReq proto.Message) (string, error) { - var mimeType string - mediaType, _, err := mime.ParseMediaType(req.Header.Get("Content-Type")) - if err != nil { - mediaType = req.Header.Get("Content-Type") - } +type healthResponse struct { + Healthy bool `json:"healthy"` + LastUpdate string `json:"lastUpdate,omitempty"` + DatabaseAge string `json:"databaseAge,omitempty"` + NextUpdate string `json:"nextUpdate,omitempty"` + FreshUntil string `json:"freshUntil,omitempty"` + LastAttempt string `json:"lastAttempt,omitempty"` + UpdatePeriod string `json:"updatePeriod"` + LastUpdateError string `json:"lastUpdateError,omitempty"` + Error string `json:"error,omitempty"` +} - alt := req.URL.Query().Get("alt") - if alt == "" { - alt = mediaType - } +const usage = `sbserver: starts a Safe Browsing API v5 local-list server. - switch alt { - case "json", mimeJSON: - mimeType = mimeJSON - case "proto", mimeProto: - mimeType = mimeProto - default: - return mimeType, errors.New("invalid interchange format") - } +Usage: %s -apikey=$APIKEY -srvaddr=0.0.0.0:80 - switch mediaType { - case mimeJSON: - if err := jsonpb.Unmarshal(req.Body, pbReq); err != nil { - return mimeType, err - } - case mimeProto: - body, err := ioutil.ReadAll(req.Body) - if err != nil { - return mimeType, err - } - if err := proto.Unmarshal(body, pbReq); err != nil { - return mimeType, err - } - } - return mimeType, nil -} +` -// marshal writes pbResp into resp. The mime can either be JSON or ProtoBuf. -func marshal(resp http.ResponseWriter, pbResp proto.Message, mime string) error { - resp.Header().Set("Content-Type", mime) - switch mime { - case mimeProto: - body, err := proto.Marshal(pbResp) - if err != nil { - return err - } - if _, err := resp.Write(body); err != nil { - return err - } - case mimeJSON: - var m jsonpb.Marshaler - var b bytes.Buffer - if err := m.Marshal(&b, pbResp); err != nil { - return err - } - if _, err := resp.Write(b.Bytes()); err != nil { - return err - } - default: - return errors.New("invalid interchange format") +func writeJSON(resp http.ResponseWriter, status int, value interface{}) { + resp.Header().Set("Content-Type", mimeJSON) + resp.WriteHeader(status) + if err := json.NewEncoder(resp).Encode(value); err != nil { + http.Error(resp, err.Error(), http.StatusInternalServerError) } - return nil } -// serveStatus writes a simple JSON with server status information to resp. func serveStatus(resp http.ResponseWriter, req *http.Request, sb *safebrowsing.SafeBrowser) { stats, sbErr := sb.Status() errStr := "" if sbErr != nil { errStr = sbErr.Error() } - buf, err := json.Marshal(struct { - Stats safebrowsing.Stats - Error string + writeJSON(resp, http.StatusOK, struct { + Stats safebrowsing.Stats `json:"stats"` + Error string `json:"error"` }{stats, errStr}) - if err != nil { - http.Error(resp, err.Error(), http.StatusInternalServerError) +} + +func serveHealth(resp http.ResponseWriter, req *http.Request, sb *safebrowsing.SafeBrowser) { + if req.Method != http.MethodGet { + http.Error(resp, "invalid method", http.StatusMethodNotAllowed) return } - resp.Header().Set("Content-Type", mimeJSON) - resp.Write(buf) + + health := sb.DatabaseHealth() + body := healthResponse{ + Healthy: health.Healthy, + UpdatePeriod: health.UpdatePeriod.String(), + LastUpdateError: health.LastUpdateError, + Error: health.Error, + } + if !health.LastUpdate.IsZero() { + body.LastUpdate = health.LastUpdate.UTC().Format(time.RFC3339) + body.DatabaseAge = health.Age.Round(time.Second).String() + } + if !health.NextUpdate.IsZero() { + body.NextUpdate = health.NextUpdate.UTC().Format(time.RFC3339) + } + if !health.FreshUntil.IsZero() { + body.FreshUntil = health.FreshUntil.UTC().Format(time.RFC3339) + } + if !health.LastAttempt.IsZero() { + body.LastAttempt = health.LastAttempt.UTC().Format(time.RFC3339) + } + + status := http.StatusOK + if !health.Healthy { + status = http.StatusServiceUnavailable + } + resp.Header().Set("Cache-Control", "no-store") + writeJSON(resp, status, body) } -// serveLookups is a light-weight implementation of the "/v4/threatMatches:find" -// API endpoint. This allows clients to look up whether a given URL is safe. -// Unlike the official API, it does not require an API key. -// It supports both JSON and ProtoBuf. func serveLookups(resp http.ResponseWriter, req *http.Request, sb *safebrowsing.SafeBrowser) { - if req.Method != "POST" { + if req.Method != http.MethodPost { http.Error(resp, "invalid method", http.StatusBadRequest) return } - // Decode the request message. - pbReq := new(pb.FindThreatMatchesRequest) - mime, err := unmarshal(req, pbReq) - if err != nil { + var searchReq urlSearchRequest + if err := json.NewDecoder(req.Body).Decode(&searchReq); err != nil { http.Error(resp, err.Error(), http.StatusBadRequest) return } - - // TODO: Should this handler use the information in threatTypes, - // platformTypes, and threatEntryTypes? - - // Parse the request message. - var urls []string - tes := pbReq.GetThreatInfo().GetThreatEntries() - for _, u := range tes { - urls = append(urls, u.Url) - if u.Url == "" || len(u.Hash) > 0 { - http.Error(resp, "only ThreatEntry.Url may be set", http.StatusBadRequest) - return - } + urls := append([]string(nil), searchReq.URLs...) + if searchReq.URL != "" { + urls = append(urls, searchReq.URL) + } + if len(urls) == 0 { + http.Error(resp, "missing url or urls", http.StatusBadRequest) + return } - // Lookup the URLs. - utss, err := sb.LookupURLsContext(req.Context(), urls) + threats, err := sb.LookupURLsContext(req.Context(), urls) if err != nil { http.Error(resp, err.Error(), http.StatusInternalServerError) return } - // Compose the response message. - pbResp := new(pb.FindThreatMatchesResponse) - for i, uts := range utss { - // Use map to condense duplicate ThreatDescriptor entries. - tdm := make(map[safebrowsing.ThreatDescriptor]bool) - for _, ut := range uts { - tdm[ut.ThreatDescriptor] = true - } - - for td := range tdm { - tm := &pb.ThreatMatch{ - Threat: &pb.ThreatEntry{Url: urls[i]}, - ThreatType: pb.ThreatType(td.ThreatType), - PlatformType: pb.PlatformType(td.PlatformType), - ThreatEntryType: pb.ThreatEntryType(td.ThreatEntryType), + var searchResp urlSearchResponse + for i, urlThreats := range threats { + seen := make(map[string]bool) + for _, threat := range urlThreats { + key := urls[i] + "\x00" + threat.ThreatType.String() + if seen[key] { + continue } - pbResp.Matches = append(pbResp.Matches, tm) + seen[key] = true + searchResp.Matches = append(searchResp.Matches, urlMatch{ + URL: urls[i], + ThreatType: threat.ThreatType.String(), + HashList: threat.Name, + Pattern: threat.Pattern, + }) } } - - // Encode the response message. - if err := marshal(resp, pbResp, mime); err != nil { - http.Error(resp, err.Error(), http.StatusInternalServerError) - return - } + writeJSON(resp, http.StatusOK, searchResp) } -// serveLists is a light-weight implementation of the "/v4/threatLists" -// API endpoint. This informs the client of which threat lists are available. -// Unlike the official API, it does not require an API key. -// It supports both JSON and ProtoBuf. func serveLists(resp http.ResponseWriter, req *http.Request, conf *safebrowsing.Config) { - var mime string - switch req.URL.Query().Get("alt") { - case "", "json": - mime = mimeJSON - case "proto": - mime = mimeProto - default: - http.Error(resp, "invalid request type", http.StatusBadRequest) - return - } - if req.Method != "GET" { + if req.Method != http.MethodGet { http.Error(resp, "invalid method", http.StatusBadRequest) return } - - tls := safebrowsing.DefaultThreatLists - if len(conf.ThreatLists) != 0 { - tls = conf.ThreatLists - } - - pbResp := new(pb.ListThreatListsResponse) - for _, td := range tls { - pbResp.ThreatLists = append(pbResp.ThreatLists, &pb.ThreatListDescriptor{ - ThreatType: pb.ThreatType(td.ThreatType), - PlatformType: pb.PlatformType(td.PlatformType), - ThreatEntryType: pb.ThreatEntryType(td.ThreatEntryType), - }) - } - - // Encode the response message. - if err := marshal(resp, pbResp, mime); err != nil { - http.Error(resp, err.Error(), http.StatusInternalServerError) - return + lists := safebrowsing.DefaultHashLists + if len(conf.HashLists) != 0 { + lists = conf.HashLists } + writeJSON(resp, http.StatusOK, struct { + HashLists []safebrowsing.HashList `json:"hashLists"` + }{lists}) } func parseTemplates(fs http.FileSystem, t *template.Template, paths ...string) (*template.Template, error) { @@ -458,11 +214,9 @@ func parseTemplates(fs http.FileSystem, t *template.Template, paths ...string) ( return t, nil } -// serveRedirector implements a basic HTTP redirector that will filter out -// redirect URLs that are unsafe according to the Safe Browsing API. func serveRedirector(resp http.ResponseWriter, req *http.Request, sb *safebrowsing.SafeBrowser, fs http.FileSystem) { rawURL := req.URL.Query().Get("url") - if rawURL == "" || req.URL.Path != "/r" { + if rawURL == "" || req.URL.Path != redirectPath { http.NotFound(resp, req) return } @@ -489,16 +243,13 @@ func serveRedirector(resp http.ResponseWriter, req *http.Request, sb *safebrowsi http.Error(resp, err.Error(), http.StatusInternalServerError) return } - err = t.Execute(resp, map[string]interface{}{ - "Threat": threat, - "Url": parsedURL}) - if err != nil { + if err := t.Execute(resp, map[string]interface{}{"Threat": threat, "Url": parsedURL}); err != nil { http.Error(resp, err.Error(), http.StatusInternalServerError) } return } } - http.Error(resp, err.Error(), http.StatusInternalServerError) + http.Error(resp, "no template for threat", http.StatusInternalServerError) } func main() { @@ -511,6 +262,7 @@ func main() { fmt.Fprintln(os.Stderr, "No -apikey specified") os.Exit(1) } + conf := safebrowsing.Config{ APIKey: *apiKeyFlag, ProxyURL: *proxyFlag, @@ -519,22 +271,25 @@ func main() { } sb, err := safebrowsing.NewSafeBrowser(conf) if err != nil { - fmt.Fprintln(os.Stderr, "Unable to initialize Safe Browsing client: ", err) + fmt.Fprintln(os.Stderr, "Unable to initialize Safe Browsing client:", err) os.Exit(1) } statikFS, err := fs.New() if err != nil { - fmt.Fprintln(os.Stderr, "Unable to initialize static files: ", err) + fmt.Fprintln(os.Stderr, "Unable to initialize static files:", err) os.Exit(1) } http.HandleFunc(statusPath, func(w http.ResponseWriter, r *http.Request) { serveStatus(w, r, sb) }) - http.HandleFunc(findThreatPath, func(w http.ResponseWriter, r *http.Request) { + http.HandleFunc(healthPath, func(w http.ResponseWriter, r *http.Request) { + serveHealth(w, r, sb) + }) + http.HandleFunc(searchURLsPath, func(w http.ResponseWriter, r *http.Request) { serveLookups(w, r, sb) }) - http.HandleFunc(getThreatListsPath, func(w http.ResponseWriter, r *http.Request) { + http.HandleFunc(getHashListsPath, func(w http.ResponseWriter, r *http.Request) { serveLists(w, r, &conf) }) http.HandleFunc(redirectPath, func(w http.ResponseWriter, r *http.Request) { @@ -545,7 +300,5 @@ func main() { fmt.Fprintln(os.Stdout, "Starting server at", *srvAddrFlag) if err := http.ListenAndServe(*srvAddrFlag, nil); err != nil { fmt.Fprintln(os.Stderr, "Server error:", err) - return } - fmt.Fprintln(os.Stdout, "Stopping server") } diff --git a/database.go b/database.go index 91bff38..747bc2f 100644 --- a/database.go +++ b/database.go @@ -5,12 +5,6 @@ // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. package safebrowsing @@ -18,6 +12,7 @@ import ( "bytes" "compress/gzip" "context" + "encoding/base64" "encoding/gob" "errors" "log" @@ -25,88 +20,71 @@ import ( "os" "sync" "time" - - pb "github.com/google/safebrowsing/internal/safebrowsing_proto" ) -// jitter is the maximum amount of time that we expect an API list update to -// actually take. We add this time to the update period time to give some -// leeway before declaring the database as stale. const ( maxRetryDelay = 24 * time.Hour baseRetryDelay = 15 * time.Minute jitter = 30 * time.Second ) -// database tracks the state of the threat lists published by the Safe Browsing -// API. Since the global blacklist is constantly changing, the contents of the -// database needs to be periodically synced with the Safe Browsing servers in -// order to provide protection for the latest threats. -// -// The process for updating the database is as follows: -// * At startup, if a database file is provided, then load it. If loaded -// properly (not corrupted and not stale), then set tfu as the contents. -// Otherwise, pull a new threat list from the Safe Browsing API. -// * Periodically, synchronize the database with the Safe Browsing API. -// This uses the State fields to update only parts of the threat list that have -// changed since the last sync. -// * Anytime tfu is updated, generate a new tfl. -// -// The process for querying the database is as follows: -// * Check if the requested full hash matches any partial hash in tfl. -// If a match is found, return a set of ThreatDescriptors with a partial match. type database struct { config *Config - - // threatsForUpdate maps ThreatDescriptors to lists of partial hashes. - // This data structure is in a format that is easily updated by the API. - // It is also the form that is written to disk. - tfu threatsForUpdate - mu sync.Mutex // Protects tfu - - // threatsForLookup maps ThreatDescriptors to sets of partial hashes. - // This data structure is in a format that is easily queried. - tfl threatsForLookup - ml sync.RWMutex // Protects tfl, err, and last - - err error // Last error encountered - readyCh chan struct{} // Used for waiting until not in an error state. - last time.Time // Last time the threat list were synced - updateAPIErrors uint // Number of times we attempted to contact the api and failed - - log *log.Logger + tfu threatsForUpdate + mu sync.Mutex + tfl threatsForLookup + ml sync.RWMutex + + err error + readyCh chan struct{} + last time.Time + next time.Time + freshUntil time.Time + lastAttempt time.Time + lastUpdateError string + updateAPIErrors uint + log *log.Logger } -type threatsForUpdate map[ThreatDescriptor]partialHashes +type threatsForUpdate map[string]partialHashes + type partialHashes struct { - // Since the Hashes field is only needed when storing to disk and when - // updating, this field is cleared except for when it is in use. - // This is done to reduce memory usage as the contents of this can be - // regenerated from the tfl. - Hashes hashPrefixes - - SHA256 []byte // The SHA256 over Hashes - State []byte // Arbitrary binary blob to synchronize state with API + Hashes hashPrefixes + SHA256 []byte + Version []byte + ThreatType ThreatType } -type threatsForLookup map[ThreatDescriptor]hashSet +type threatsForLookup map[string]hashSet -// databaseFormat is a light struct used only for gob encoding and decoding. -// As written to disk, the format of the database file is basically the gzip -// compressed version of the gob encoding of databaseFormat. type databaseFormat struct { - Table threatsForUpdate - Time time.Time + Table threatsForUpdate + Time time.Time + NextUpdate time.Time + FreshUntil time.Time +} + +type DatabaseHealth struct { + LastUpdate time.Time + NextUpdate time.Time + FreshUntil time.Time + LastAttempt time.Time + LastUpdateError string + UpdatePeriod time.Duration + Age time.Duration + Healthy bool + Error string } -// Init initializes the database from the specified file in config.DBPath. -// It reports true if the database was successfully loaded. func (db *database) Init(config *Config, logger *log.Logger) bool { db.mu.Lock() defer db.mu.Unlock() - db.setError(errors.New("not intialized")) + db.setError(errors.New("not initialized")) db.config = config db.log = logger + if db.readyCh == nil { + db.readyCh = make(chan struct{}) + } if db.config.DBPath == "" { db.log.Printf("no database file specified") db.setError(errors.New("no database loaded")) @@ -118,112 +96,132 @@ func (db *database) Init(config *Config, logger *log.Logger) bool { db.setError(err) return false } - // Validate that the database threat list stored on disk is not too stale. - if db.isStale(dbf.Time) { + nextUpdate := dbf.NextUpdate + freshUntil := dbf.FreshUntil + if nextUpdate.IsZero() { + nextUpdate = dbf.Time.Add(db.config.UpdatePeriod) + } + if freshUntil.IsZero() { + freshUntil = nextUpdate.Add(db.freshnessGrace()) + } + if db.isStale(freshUntil) { db.log.Printf("database loaded is stale") db.ml.Lock() defer db.ml.Unlock() db.setStale() return false } - // Validate that the database threat list stored on disk is at least a - // superset of the specified configuration. + tfuNew := make(threatsForUpdate) - for _, td := range db.config.ThreatLists { - if row, ok := dbf.Table[td]; ok { - tfuNew[td] = row - } else { - db.log.Printf("database configuration mismatch, missing %v", td) + for _, list := range db.config.HashLists { + row, ok := dbf.Table[list.Name] + if !ok { + db.log.Printf("database configuration mismatch, missing %v", list.Name) db.setError(errors.New("database configuration mismatch")) return false } + row.ThreatType = list.ThreatType + tfuNew[list.Name] = row } db.tfu = tfuNew - db.generateThreatsForLookups(dbf.Time) + db.generateThreatsForLookups(dbf.Time, nextUpdate, freshUntil) return true } -// Status reports the health of the database. The database is considered faulted -// if there was an error during update or if the last update has gone stale. If -// in a faulted state, the db may repair itself on the next Update. func (db *database) Status() error { db.ml.RLock() - defer db.ml.RUnlock() - if db.err != nil { - return db.err + err := db.err + db.ml.RUnlock() + return err } - if db.isStale(db.last) { + stale := db.isStale(db.freshUntil) + db.ml.RUnlock() + if stale { + db.ml.Lock() db.setStale() - return db.err + err := db.err + db.ml.Unlock() + return err } return nil } -// UpdateLag reports the amount of time in between when we expected to run -// a database update and the current time func (db *database) UpdateLag() time.Duration { - lag := db.SinceLastUpdate() - if lag < db.config.UpdatePeriod { + db.ml.RLock() + next := db.next + db.ml.RUnlock() + if next.IsZero() || db.config.now().Before(next) { return 0 } - return lag - db.config.UpdatePeriod + return db.config.now().Sub(next) } -// SinceLastUpdate gives the duration since the last database update func (db *database) SinceLastUpdate() time.Duration { db.ml.RLock() defer db.ml.RUnlock() - + if db.last.IsZero() { + return 0 + } return db.config.now().Sub(db.last) } -// Ready returns a channel that's closed when the database is ready for queries. +func (db *database) UntilNextUpdate() time.Duration { + db.ml.RLock() + defer db.ml.RUnlock() + if db.next.IsZero() || !db.config.now().Before(db.next) { + return 0 + } + return db.next.Sub(db.config.now()) +} + +func (db *database) Health() DatabaseHealth { + err := db.Status() + now := db.config.now() + + db.ml.RLock() + health := DatabaseHealth{ + LastUpdate: db.last, + NextUpdate: db.next, + FreshUntil: db.freshUntil, + LastAttempt: db.lastAttempt, + LastUpdateError: db.lastUpdateError, + UpdatePeriod: db.config.UpdatePeriod, + Healthy: err == nil, + } + db.ml.RUnlock() + if !health.LastUpdate.IsZero() { + health.Age = now.Sub(health.LastUpdate) + } + if err != nil { + health.Error = err.Error() + } + return health +} + func (db *database) Ready() <-chan struct{} { return db.readyCh } -// Update synchronizes the local threat lists with those maintained by the -// global Safe Browsing API servers. If the update is successful, Status should -// report a nil error. func (db *database) Update(ctx context.Context, api api) (time.Duration, bool) { db.mu.Lock() defer db.mu.Unlock() - // Construct the request. - var numTypes int - var s []*pb.FetchThreatListUpdatesRequest_ListUpdateRequest - for _, td := range db.config.ThreatLists { - var state []byte - if row, ok := db.tfu[td]; ok { - state = row.State + names := make([]string, 0, len(db.config.HashLists)) + versions := make(map[string][]byte) + for _, list := range db.config.HashLists { + names = append(names, list.Name) + if row, ok := db.tfu[list.Name]; ok { + versions[list.Name] = row.Version } + } - s = append(s, &pb.FetchThreatListUpdatesRequest_ListUpdateRequest{ - ThreatType: pb.ThreatType(td.ThreatType), - PlatformType: pb.PlatformType(td.PlatformType), - ThreatEntryType: pb.ThreatEntryType(td.ThreatEntryType), - Constraints: &pb.FetchThreatListUpdatesRequest_ListUpdateRequest_Constraints{ - SupportedCompressions: db.config.compressionTypes}, - State: state, - }) - numTypes++ - } - req := &pb.FetchThreatListUpdatesRequest{ - Client: &pb.ClientInfo{ - ClientId: db.config.ID, - ClientVersion: db.config.Version, - }, - ListUpdateRequests: s, - } - - // Query the API for the threat list and update the database. last := db.config.now() - resp, err := api.ListUpdate(ctx, req) + db.recordUpdateAttempt(last) + resp, err := api.ListUpdate(ctx, names, versions) if err != nil { db.log.Printf("ListUpdate failure (%d): %v", db.updateAPIErrors+1, err) - db.setError(err) - // backoff strategy: MIN((2**N-1 * 15 minutes) * (RAND + 1), 24 hours) + db.recordUpdateFailure(err) n := 1 << db.updateAPIErrors delay := time.Duration(float64(n) * (rand.Float64() + 1) * float64(baseRetryDelay)) if delay > maxRetryDelay { @@ -234,166 +232,222 @@ func (db *database) Update(ctx context.Context, api api) (time.Duration, bool) { } db.updateAPIErrors = 0 - // add jitter to wait time to avoid all servers lining up - nextUpdateWait := db.config.UpdatePeriod + time.Duration(rand.Int31n(60)-30)*time.Second - if resp.MinimumWaitDuration != nil { - serverMinWait := time.Duration(resp.MinimumWaitDuration.Seconds)*time.Second + time.Duration(resp.MinimumWaitDuration.Nanos) - if serverMinWait > nextUpdateWait { - nextUpdateWait = serverMinWait - db.log.Printf("Server requested next update in %v", nextUpdateWait) + if err := validateHashListResponse(names, resp); err != nil { + db.log.Printf("invalid ListUpdate response: %v", err) + db.recordUpdateFailure(err) + return baseRetryDelay, false + } + + var nextUpdateWait time.Duration + for _, hl := range resp.HashLists { + wait := parseDuration(hl.MinimumWaitDuration) + db.log.Printf( + "Google hash list response: name=%s update=%s minimum_wait_duration=%v additions=%d removals=%d version=%s", + hl.Name, + hashListUpdateType(hl), + wait, + riceValueCount(hl.AdditionsFourBytes), + riceValueCount(hl.CompressedRemovals), + abbreviateOpaqueValue(hl.Version), + ) + if wait > nextUpdateWait { + nextUpdateWait = wait } } - if len(resp.ListUpdateResponses) != numTypes { - db.setError(errors.New("safebrowsing: threat list count mismatch")) - db.log.Printf("invalid server response: got %d, want %d threat lists", - len(resp.ListUpdateResponses), numTypes) - return nextUpdateWait, false + if nextUpdateWait > 0 { + db.log.Printf("Google minimum_wait_duration: %v", nextUpdateWait) + } else { + db.log.Printf("Google minimum_wait_duration missing or zero; scheduling immediate follow-up update") } + db.log.Printf("Selected next database update in %v", nextUpdateWait) - // Update the threat database with the response. db.generateThreatsForUpdate() + if db.tfu == nil { + db.tfu = make(threatsForUpdate) + } + + for _, list := range db.config.HashLists { + row := db.tfu[list.Name] + row.ThreatType = list.ThreatType + db.tfu[list.Name] = row + } if err := db.tfu.update(resp); err != nil { - db.setError(err) db.log.Printf("update failure: %v", err) + db.recordUpdateFailure(err) db.tfu = nil return nextUpdateWait, false } - dbf := databaseFormat{make(threatsForUpdate), last} - for td, phs := range db.tfu { - // Copy of partialHashes before generateThreatsForLookups clobbers it. - dbf.Table[td] = phs + + nextUpdate := last.Add(nextUpdateWait) + freshUntil := nextUpdate.Add(db.freshnessGrace()) + dbf := databaseFormat{ + Table: make(threatsForUpdate), + Time: last, + NextUpdate: nextUpdate, + FreshUntil: freshUntil, + } + for name, phs := range db.tfu { + dbf.Table[name] = phs } - db.generateThreatsForLookups(last) + db.generateThreatsForLookups(last, nextUpdate, freshUntil) - // Regenerate the database and store it. if db.config.DBPath != "" { - // Semantically, we ignore save errors, but we do log them. if err := saveDatabase(db.config.DBPath, dbf); err != nil { db.log.Printf("save failure: %v", err) } } - return nextUpdateWait, true } -// Lookup looks up the full hash in the threat list and returns a partial -// hash and a set of ThreatDescriptors that may match the full hash. -func (db *database) Lookup(hash hashPrefix) (h hashPrefix, tds []ThreatDescriptor) { +func validateHashListResponse(names []string, resp *hashListsResponse) error { + if resp == nil { + return errors.New("safebrowsing: nil hash list response") + } + if len(resp.HashLists) != len(names) { + return errors.New("safebrowsing: hash list response count mismatch") + } + for i, name := range names { + if resp.HashLists[i].Name != name { + return errors.New("safebrowsing: hash list response order mismatch") + } + } + return nil +} + +func hashListUpdateType(hl hashList) string { + if hl.PartialUpdate { + return "partial" + } + return "full" +} + +func riceValueCount(values *rice32) int64 { + if values == nil { + return 0 + } + return int64(values.EntriesCount) + 1 +} + +func abbreviateOpaqueValue(value string) string { + const maxLength = 12 + if value == "" { + return "" + } + if len(value) <= maxLength { + return value + } + return value[:maxLength] + "..." +} + +func (db *database) Lookup(hash hashPrefix) (hashPrefix, []HashList) { if !hash.IsFull() { panic("hash is not full") } + var h hashPrefix + var lists []HashList db.ml.RLock() - for td, hs := range db.tfl { - if n := hs.Lookup(hash); n > 0 { - h = hash[:n] - tds = append(tds, td) + for _, list := range db.config.HashLists { + if hs, ok := db.tfl[list.Name]; ok { + if n := hs.Lookup(hash); n > 0 { + h = hash[:n] + lists = append(lists, list) + } } } db.ml.RUnlock() - return h, tds + return h, lists } -// setError clears the database state and sets the last error to be err. -// -// This assumes that the db.mu lock is already held. func (db *database) setError(err error) { db.tfu = nil - db.ml.Lock() if db.err == nil { db.readyCh = make(chan struct{}) } db.tfl, db.err, db.last = nil, err, time.Time{} + db.next, db.freshUntil = time.Time{}, time.Time{} db.ml.Unlock() } -// isStale checks whether the last successful update should be considered stale. -// Staleness is defined as being older than two of the configured update periods -// plus jitter. -func (db *database) isStale(lastUpdate time.Time) bool { - if db.config.now().Sub(lastUpdate) > 2*(db.config.UpdatePeriod+jitter) { - return true - } - return false +func (db *database) isStale(freshUntil time.Time) bool { + return freshUntil.IsZero() || db.config.now().After(freshUntil) +} + +func (db *database) freshnessGrace() time.Duration { + return db.config.UpdatePeriod + jitter } -// setStale sets the error state to a stale message, without clearing -// the database state. -// -// This assumes that the db.ml lock is already held. func (db *database) setStale() { if db.err == nil { db.readyCh = make(chan struct{}) } - db.err = errStale + if db.lastUpdateError != "" { + db.err = errors.New(errStale.Error() + ": last update error: " + db.lastUpdateError) + } else { + db.err = errStale + } } -// clearError clears the db error state, and unblocks any callers of -// WaitUntilReady. -// -// This assumes that the db.mu lock is already held. func (db *database) clearError() { db.ml.Lock() defer db.ml.Unlock() - if db.err != nil { close(db.readyCh) } db.err = nil + db.lastUpdateError = "" +} + +func (db *database) recordUpdateAttempt(at time.Time) { + db.ml.Lock() + db.lastAttempt = at + db.ml.Unlock() +} + +func (db *database) recordUpdateFailure(err error) { + db.ml.Lock() + db.lastUpdateError = err.Error() + db.ml.Unlock() } -// generateThreatsForUpdate regenerates the threatsForUpdate hashes from -// the threatsForLookup. We do this to avoid holding onto the hash lists for -// a long time, needlessly occupying lots of memory. -// -// This assumes that the db.mu lock is already held. func (db *database) generateThreatsForUpdate() { if db.tfu == nil { db.tfu = make(threatsForUpdate) } - db.ml.RLock() - for td, hs := range db.tfl { - phs := db.tfu[td] + for name, hs := range db.tfl { + phs := db.tfu[name] phs.Hashes = hs.Export() - db.tfu[td] = phs + db.tfu[name] = phs } db.ml.RUnlock() } -// generateThreatsForLookups regenerates the threatsForLookup data structure -// from the threatsForUpdate data structure and stores the last timestamp. -// Since the hashes are effectively stored as a set inside the threatsForLookup, -// we clear out the hashes slice in threatsForUpdate so that it can be GCed. -// -// This assumes that the db.mu lock is already held. -func (db *database) generateThreatsForLookups(last time.Time) { +func (db *database) generateThreatsForLookups(last, next, freshUntil time.Time) { tfl := make(threatsForLookup) - for td, phs := range db.tfu { + for name, phs := range db.tfu { var hs hashSet hs.Import(phs.Hashes) - tfl[td] = hs - - phs.Hashes = nil // Clear hashes to keep memory usage low - db.tfu[td] = phs + tfl[name] = hs + phs.Hashes = nil + db.tfu[name] = phs } db.ml.Lock() wasBad := db.err != nil db.tfl, db.last = tfl, last + db.next, db.freshUntil = next, freshUntil + db.lastUpdateError = "" db.ml.Unlock() - if wasBad { db.clearError() db.log.Printf("database is now healthy") } } -// saveDatabase saves the database threat list to a file. func saveDatabase(path string, db databaseFormat) (err error) { - var file *os.File - file, err = os.Create(path) + file, err := os.Create(path) if err != nil { return err } @@ -412,18 +466,11 @@ func saveDatabase(path string, db databaseFormat) (err error) { err = zerr } }() - - encoder := gob.NewEncoder(gz) - if err = encoder.Encode(db); err != nil { - return err - } - return nil + return gob.NewEncoder(gz).Encode(db) } -// loadDatabase loads the database state from a file. func loadDatabase(path string) (db databaseFormat, err error) { - var file *os.File - file, err = os.Open(path) + file, err := os.Open(path) if err != nil { return db, err } @@ -443,8 +490,7 @@ func loadDatabase(path string) (db databaseFormat, err error) { } }() - decoder := gob.NewDecoder(gz) - if err = decoder.Decode(&db); err != nil { + if err = gob.NewDecoder(gz).Decode(&db); err != nil { return db, err } for _, dv := range db.Table { @@ -455,50 +501,28 @@ func loadDatabase(path string) (db databaseFormat, err error) { return db, nil } -// update updates the threat list according to the API response. -func (tfu threatsForUpdate) update(resp *pb.FetchThreatListUpdatesResponse) error { - // For each update response do the removes and adds - for _, m := range resp.GetListUpdateResponses() { - td := ThreatDescriptor{ - PlatformType: PlatformType(m.PlatformType), - ThreatType: ThreatType(m.ThreatType), - ThreatEntryType: ThreatEntryType(m.ThreatEntryType), +func (tfu threatsForUpdate) update(resp *hashListsResponse) error { + for _, hl := range resp.HashLists { + phs := tfu[hl.Name] + if !hl.PartialUpdate { + phs.Hashes = nil + phs.SHA256 = nil + } else if _, ok := tfu[hl.Name]; !ok { + return errors.New("safebrowsing: partial update received for non-existent key") } - phs, ok := tfu[td] - switch m.ResponseType { - case pb.FetchThreatListUpdatesResponse_ListUpdateResponse_PARTIAL_UPDATE: - if !ok { - return errors.New("safebrowsing: partial update received for non-existent key") - } - case pb.FetchThreatListUpdatesResponse_ListUpdateResponse_FULL_UPDATE: - if len(m.Removals) > 0 { - return errors.New("safebrowsing: indices to be removed included in a full update") - } - phs = partialHashes{} - default: - return errors.New("safebrowsing: unknown response type") - } - - // Hashes must be sorted for removal logic to work properly. phs.Hashes.Sort() - - for _, removal := range m.Removals { - idxs, err := decodeIndices(removal) + if hl.CompressedRemovals != nil { + idxs, err := decodeRice32(hl.CompressedRemovals) if err != nil { return err } - for _, i := range idxs { - if i < 0 || i >= int32(len(phs.Hashes)) { + if i >= uint32(len(phs.Hashes)) { return errors.New("safebrowsing: invalid removal index") } phs.Hashes[i] = "" } - } - - // If any removal was performed, compact the list of hashes. - if len(m.Removals) > 0 { compactHashes := phs.Hashes[:0] for _, h := range phs.Hashes { if h != "" { @@ -508,29 +532,36 @@ func (tfu threatsForUpdate) update(resp *pb.FetchThreatListUpdatesResponse) erro phs.Hashes = compactHashes } - for _, addition := range m.Additions { - hashes, err := decodeHashes(addition) + if hl.AdditionsFourBytes != nil { + hashes, err := decodeRice32Hashes(hl.AdditionsFourBytes) if err != nil { return err } phs.Hashes = append(phs.Hashes, hashes...) } - // Hashes must be sorted for SHA256 checksum to be correct. phs.Hashes.Sort() if err := phs.Hashes.Validate(); err != nil { return err } - - if cs := m.GetChecksum(); cs != nil { - phs.SHA256 = cs.Sha256 + if hl.SHA256Checksum != "" { + checksum, err := base64.StdEncoding.DecodeString(hl.SHA256Checksum) + if err != nil { + return err + } + phs.SHA256 = checksum } - if !bytes.Equal(phs.SHA256, phs.Hashes.SHA256()) { + if len(phs.SHA256) > 0 && !bytes.Equal(phs.SHA256, phs.Hashes.SHA256()) { return errors.New("safebrowsing: threat list SHA256 mismatch") } - - phs.State = m.NewClientState - tfu[td] = phs + if hl.Version != "" { + version, err := base64.StdEncoding.DecodeString(hl.Version) + if err != nil { + return err + } + phs.Version = version + } + tfu[hl.Name] = phs } return nil } diff --git a/database_test.go b/database_test.go deleted file mode 100644 index eb8e97b..0000000 --- a/database_test.go +++ /dev/null @@ -1,843 +0,0 @@ -// Copyright 2016 Google Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package safebrowsing - -import ( - "context" - "encoding/hex" - "errors" - "io" - "io/ioutil" - "log" - "math" - "os" - "reflect" - "sort" - "strings" - "testing" - "time" - - google_protobuf "github.com/golang/protobuf/ptypes/duration" - pb "github.com/google/safebrowsing/internal/safebrowsing_proto" -) - -func mustGetTempFile(t *testing.T) string { - f, err := ioutil.TempFile("", "") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - path := f.Name() - if err := f.Close(); err != nil { - t.Fatalf("unexpected error: %v", err) - } - return path -} - -func mustDecodeHex(t *testing.T, s string) []byte { - buf, err := hex.DecodeString(s) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - return buf -} - -func newHashSet(phs hashPrefixes) (hs hashSet) { - hs.Import(phs) - return hs -} - -func TestDatabaseInit(t *testing.T) { - path := mustGetTempFile(t) - defer os.Remove(path) - - now := time.Unix(1451436338, 951473000) - mockNow := func() time.Time { return now } - - vectors := []struct { - config *Config // Input configuration - oldDB *database // The old database (before export) - newDB *database // The expected new database (after import) - fail bool // Expected failure - }{{ - // Load from a valid database file. - config: &Config{ - ThreatLists: []ThreatDescriptor{ - {0, 2, 3}, - {1, 2, 3}, - }, - UpdatePeriod: DefaultUpdatePeriod, - }, - oldDB: &database{ - last: now.Add(-DefaultUpdatePeriod + time.Minute), - tfu: threatsForUpdate{ - {0, 2, 3}: partialHashes{ - Hashes: []hashPrefix{"aaaa", "bbbb"}, - SHA256: mustDecodeHex(t, "e5c1edb50ff8b4fcc3ead3a845ffbe1ad51c9dae5d44335a5c333b57ac8df062"), - State: []byte("state1"), - }, - {1, 2, 3}: partialHashes{ - Hashes: []hashPrefix{"bbbb", "cccc"}, - SHA256: mustDecodeHex(t, "9a720c6ee500f5a0d4e5477fc9f3d8573226723d0b338b0c8f572d877bdfa224"), - State: []byte("state2"), - }, - }, - }, - newDB: &database{ - last: now.Add(-DefaultUpdatePeriod + time.Minute), - tfu: threatsForUpdate{ - {0, 2, 3}: partialHashes{ - SHA256: mustDecodeHex(t, "e5c1edb50ff8b4fcc3ead3a845ffbe1ad51c9dae5d44335a5c333b57ac8df062"), - State: []byte("state1"), - }, - {1, 2, 3}: partialHashes{ - SHA256: mustDecodeHex(t, "9a720c6ee500f5a0d4e5477fc9f3d8573226723d0b338b0c8f572d877bdfa224"), - State: []byte("state2"), - }, - }, - tfl: threatsForLookup{ - {0, 2, 3}: newHashSet([]hashPrefix{"aaaa", "bbbb"}), - {1, 2, 3}: newHashSet([]hashPrefix{"bbbb", "cccc"}), - }, - }, - }, { - // Load from an older but not yet stale valid database file. - config: &Config{ - ThreatLists: []ThreatDescriptor{ - {0, 2, 3}, - {1, 2, 3}, - }, - UpdatePeriod: DefaultUpdatePeriod, - }, - oldDB: &database{ - last: now.Add(-DefaultUpdatePeriod + (30 * time.Minute)), - tfu: threatsForUpdate{ - {0, 2, 3}: partialHashes{ - Hashes: []hashPrefix{"aaaa", "bbbb"}, - SHA256: mustDecodeHex(t, "e5c1edb50ff8b4fcc3ead3a845ffbe1ad51c9dae5d44335a5c333b57ac8df062"), - State: []byte("state1"), - }, - {1, 2, 3}: partialHashes{ - Hashes: []hashPrefix{"bbbb", "cccc"}, - SHA256: mustDecodeHex(t, "9a720c6ee500f5a0d4e5477fc9f3d8573226723d0b338b0c8f572d877bdfa224"), - State: []byte("state2"), - }, - }, - }, - newDB: &database{ - last: now.Add(-DefaultUpdatePeriod + (30 * time.Minute)), - tfu: threatsForUpdate{ - {0, 2, 3}: partialHashes{ - SHA256: mustDecodeHex(t, "e5c1edb50ff8b4fcc3ead3a845ffbe1ad51c9dae5d44335a5c333b57ac8df062"), - State: []byte("state1"), - }, - {1, 2, 3}: partialHashes{ - SHA256: mustDecodeHex(t, "9a720c6ee500f5a0d4e5477fc9f3d8573226723d0b338b0c8f572d877bdfa224"), - State: []byte("state2"), - }, - }, - tfl: threatsForLookup{ - {0, 2, 3}: newHashSet([]hashPrefix{"aaaa", "bbbb"}), - {1, 2, 3}: newHashSet([]hashPrefix{"bbbb", "cccc"}), - }, - }, - }, { - // Load from a valid database file with more descriptors than in configuration. - config: &Config{ - ThreatLists: []ThreatDescriptor{{0, 2, 3}}, - }, - oldDB: &database{ - last: now, - tfu: threatsForUpdate{ - {0, 2, 3}: partialHashes{ - Hashes: []hashPrefix{"aaaa", "bbbb"}, - SHA256: mustDecodeHex(t, "e5c1edb50ff8b4fcc3ead3a845ffbe1ad51c9dae5d44335a5c333b57ac8df062"), - State: []byte("state1"), - }, - {1, 2, 3}: partialHashes{ - Hashes: []hashPrefix{"bbbb", "cccc"}, - SHA256: mustDecodeHex(t, "9a720c6ee500f5a0d4e5477fc9f3d8573226723d0b338b0c8f572d877bdfa224"), - State: []byte("state2"), - }, - {3, 2, 3}: partialHashes{ - Hashes: []hashPrefix{"xxx", "yyy", "zzz"}, - SHA256: mustDecodeHex(t, "cc6c955cadf2cc09442c0848ce8e165b8f9aa5974916de7186a9e1b6c4e7937e"), - }, - }, - }, - newDB: &database{ - last: now, - tfu: threatsForUpdate{ - {0, 2, 3}: partialHashes{ - SHA256: mustDecodeHex(t, "e5c1edb50ff8b4fcc3ead3a845ffbe1ad51c9dae5d44335a5c333b57ac8df062"), - State: []byte("state1"), - }, - }, - tfl: threatsForLookup{ - {0, 2, 3}: newHashSet([]hashPrefix{"aaaa", "bbbb"}), - }, - }, - }, { - // Load from a invalid database file with fewer descriptors than in configuration. - config: &Config{ - ThreatLists: []ThreatDescriptor{ - {0, 1, 3}, - {0, 2, 3}, - {1, 1, 3}, - {1, 2, 3}, - }, - }, - oldDB: &database{ - last: now, - tfu: threatsForUpdate{ - {0, 2, 3}: partialHashes{ - Hashes: []hashPrefix{"aaaa", "bbbb"}, - SHA256: mustDecodeHex(t, "e5c1edb50ff8b4fcc3ead3a845ffbe1ad51c9dae5d44335a5c333b57ac8df062"), - State: []byte("state1"), - }, - {1, 2, 3}: partialHashes{ - Hashes: []hashPrefix{"bbbb", "cccc"}, - SHA256: mustDecodeHex(t, "9a720c6ee500f5a0d4e5477fc9f3d8573226723d0b338b0c8f572d877bdfa224"), - State: []byte("state2"), - }, - {0, 1, 3}: partialHashes{ - Hashes: []hashPrefix{"xxx", "yyy", "zzz"}, - SHA256: mustDecodeHex(t, "cc6c955cadf2cc09442c0848ce8e165b8f9aa5974916de7186a9e1b6c4e7937e"), - }, - }, - }, - fail: true, - }, { - // Load from a stale database file. - config: &Config{ - ThreatLists: []ThreatDescriptor{ - {0, 2, 3}, - {1, 2, 3}, - }, - UpdatePeriod: DefaultUpdatePeriod, - }, - oldDB: &database{ - last: now.Add(-2 * (DefaultUpdatePeriod + time.Minute)), - tfu: threatsForUpdate{ - {0, 2, 3}: partialHashes{ - Hashes: []hashPrefix{"aaaa", "bbbb"}, - SHA256: mustDecodeHex(t, "e5c1edb50ff8b4fcc3ead3a845ffbe1ad51c9dae5d44335a5c333b57ac8df062"), - State: []byte("state1"), - }, - {1, 2, 3}: partialHashes{ - Hashes: []hashPrefix{"bbbb", "cccc"}, - SHA256: mustDecodeHex(t, "9a720c6ee500f5a0d4e5477fc9f3d8573226723d0b338b0c8f572d877bdfa224"), - State: []byte("state2"), - }, - }, - }, - fail: true, - }, { - // Load from a corrupted database file (has bad SHA256 checksums). - config: &Config{ - ThreatLists: []ThreatDescriptor{ - {0, 2, 3}, - {1, 2, 3}, - }, - }, - oldDB: &database{ - last: now, - tfu: threatsForUpdate{ - {0, 2, 3}: partialHashes{ - Hashes: []hashPrefix{"aaaa", "bbbb"}, - State: []byte("state1"), - SHA256: []byte("bad checksum"), - }, - }, - }, - fail: true, - }} - - logger := log.New(ioutil.Discard, "", 0) - for i, v := range vectors { - v.config.DBPath = path - - db1 := v.oldDB - db1.config = v.config - dbf := databaseFormat{db1.tfu, db1.last} - if err := saveDatabase(db1.config.DBPath, dbf); err != nil { - t.Errorf("test %d, unexpected save error: %v", i, err) - } - - db2 := new(database) - v.config.now = mockNow - if fail := !db2.Init(v.config, logger); fail != v.fail { - t.Errorf("test %d, mismatching status: got %v, want %v", i, fail, v.fail) - } - - db2.config, db2.log, db2.readyCh = nil, nil, nil - if !v.fail && !reflect.DeepEqual(db2, v.newDB) { - t.Errorf("test %d, mismatching database contents:\ngot %+v\nwant %+v", i, db2, v.newDB) - } - } -} - -func TestDatabaseUpdate(t *testing.T) { - var ( - td013 = ThreatDescriptor{0, 1, 3} - td014 = ThreatDescriptor{0, 1, 4} - td113 = ThreatDescriptor{1, 1, 3} - td114 = ThreatDescriptor{1, 1, 4} - - full = int(pb.FetchThreatListUpdatesResponse_ListUpdateResponse_FULL_UPDATE) - partial = int(pb.FetchThreatListUpdatesResponse_ListUpdateResponse_PARTIAL_UPDATE) - - config = &Config{ - ThreatLists: []ThreatDescriptor{ - {0, 2, 3}, - {0, 2, 4}, - {1, 2, 3}, - {1, 2, 4}, - }, - UpdatePeriod: 1800 * time.Second, - } - logger = log.New(ioutil.Discard, "", 0) - ) - - // Helper function to aid in the construction on responses. - newResp := func(td ThreatDescriptor, rtype int, dels []int32, adds []string, state string, chksum string) *pb.FetchThreatListUpdatesResponse_ListUpdateResponse { - resp := &pb.FetchThreatListUpdatesResponse_ListUpdateResponse{ - ThreatType: pb.ThreatType(td.ThreatType), - PlatformType: pb.PlatformType(td.PlatformType), - ThreatEntryType: pb.ThreatEntryType(td.ThreatEntryType), - ResponseType: pb.FetchThreatListUpdatesResponse_ListUpdateResponse_ResponseType(rtype), - NewClientState: []byte(state), - Checksum: &pb.Checksum{Sha256: mustDecodeHex(t, chksum)}, - } - if dels != nil { - resp.Removals = []*pb.ThreatEntrySet{{ - CompressionType: pb.CompressionType_RAW, - RawIndices: &pb.RawIndices{Indices: dels}, - }} - } - if adds != nil { - bySize := make(map[int][]string) - for _, s := range adds { - bySize[len(s)] = append(bySize[len(s)], s) - } - for n, hs := range bySize { - sort.Strings(hs) - resp.Additions = append(resp.Additions, &pb.ThreatEntrySet{ - CompressionType: pb.CompressionType_RAW, - RawHashes: &pb.RawHashes{ - PrefixSize: int32(n), - RawHashes: []byte(strings.Join(hs, "")), - }, - }) - } - } - return resp - } - - // Setup mocking objects. - var now time.Time - mockNow := func() time.Time { return now } - config.now = mockNow - - var resp pb.FetchThreatListUpdatesResponse - var errResponse error - mockAPI := &mockAPI{ - listUpdate: func(context.Context, *pb.FetchThreatListUpdatesRequest) (*pb.FetchThreatListUpdatesResponse, error) { - return &resp, errResponse - }, - } - - // Setup the database under test. - var gotDB, wantDB *database - db := &database{config: config, log: logger} - - // Update 0: partial update on empty database. - now = now.Add(time.Hour) - resp = pb.FetchThreatListUpdatesResponse{ - ListUpdateResponses: []*pb.FetchThreatListUpdatesResponse_ListUpdateResponse{ - newResp(td013, full, nil, nil, - "a0", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"), - newResp(td113, full, nil, nil, - "b0", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"), - newResp(td014, full, nil, nil, - "c0", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"), - newResp(td114, partial, []int32{0, 1, 2, 3}, nil, - "d0", "0000000000000000000000000000000000000000000000000000000000000000"), - }, - MinimumWaitDuration: &google_protobuf.Duration{Seconds: 1200}, - } - delay, updated := db.Update(context.Background(), mockAPI) - if db.err == nil || updated { - t.Fatalf("update 0, unexpected update success") - } - - // MinimumWaitDuration is less than the jitter generated by the db code so don't use it - if math.Abs((config.UpdatePeriod - delay).Seconds()) > 31 { - t.Fatalf("update 0, jitter was more than 30 seconds") - } - - // Update 1: full update to all values. - now = now.Add(time.Hour) - resp = pb.FetchThreatListUpdatesResponse{ - ListUpdateResponses: []*pb.FetchThreatListUpdatesResponse_ListUpdateResponse{ - newResp(td013, full, nil, []string{"aaaa", "bbbb", "cccc", "0421e", "0421f", "a64392f6f89487"}, - "a1", "6a2480447ff0715d5c28090c3333fba69bd918faad4e9e0f977f3cc76f422ad6"), - newResp(td113, full, nil, nil, - "b1", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"), - newResp(td014, full, nil, []string{"aaaa", "bbbb", "cccc", "dddd"}, - "c1", "147eb9dcde0e090429c01dbf634fd9b69a7f141f005c387a9c00498908499dde"), - newResp(td114, full, nil, []string{"aaaa", "0421e", "666666", "7777777", "88888888"}, - "d1", "a3b93fac424834c2447e2dbe5db3ec8553519777523907ea310e207f556a7637"), - }, - MinimumWaitDuration: &google_protobuf.Duration{Seconds: 2000}, - } - delay, updated = db.Update(context.Background(), mockAPI) - if db.err != nil || !updated { - t.Fatalf("update 1, unexpected update error: %v", db.err) - } - - // Make sure we respect the MinimumWaitDuration from the API - expectedDelay := time.Duration(2000 * time.Second) - if delay != expectedDelay { - t.Fatalf("update 1, expected delay %v got %v", expectedDelay, delay) - } - - gotDB = &database{last: db.last, tfu: db.tfu, tfl: db.tfl} - wantDB = &database{ - last: now, - tfu: threatsForUpdate{ - td013: {SHA256: gotDB.tfu[td013].SHA256, State: []byte{0x61, 0x31}}, - td113: {SHA256: gotDB.tfu[td113].SHA256, State: []byte{0x62, 0x31}}, - td014: {SHA256: gotDB.tfu[td014].SHA256, State: []byte{0x63, 0x31}}, - td114: {SHA256: gotDB.tfu[td114].SHA256, State: []byte{0x64, 0x31}}, - }, - tfl: threatsForLookup{ - td013: newHashSet([]hashPrefix{"0421e", "0421f", "a64392f6f89487", "aaaa", "bbbb", "cccc"}), - td113: newHashSet([]hashPrefix{}), - td014: newHashSet([]hashPrefix{"aaaa", "bbbb", "cccc", "dddd"}), - td114: newHashSet([]hashPrefix{"0421e", "666666", "7777777", "88888888", "aaaa"}), - }, - } - if !reflect.DeepEqual(gotDB.tfu, wantDB.tfu) { - t.Errorf("update 1, threats for update mismatch:\ngot %+v\nwant %+v", gotDB.tfu, wantDB.tfu) - } - if !reflect.DeepEqual(gotDB.tfl, wantDB.tfl) { - t.Fatalf("update 1, threats for lookup mismatch:\ngot %+v\nwant %+v", gotDB.tfl, wantDB.tfl) - } - - // Update 2: partial update with no changes. - now = now.Add(time.Hour) - resp = pb.FetchThreatListUpdatesResponse{ - ListUpdateResponses: []*pb.FetchThreatListUpdatesResponse_ListUpdateResponse{ - newResp(td013, partial, nil, nil, - "a1", "6a2480447ff0715d5c28090c3333fba69bd918faad4e9e0f977f3cc76f422ad6"), - newResp(td113, partial, nil, nil, - "b1", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"), - newResp(td014, partial, nil, nil, - "c1", "147eb9dcde0e090429c01dbf634fd9b69a7f141f005c387a9c00498908499dde"), - newResp(td114, partial, nil, nil, - "d1", "a3b93fac424834c2447e2dbe5db3ec8553519777523907ea310e207f556a7637"), - }, - } - delay, updated = db.Update(context.Background(), mockAPI) - if db.err != nil || !updated { - t.Fatalf("update 2, unexpected update error: %v", db.err) - } - if math.Abs((config.UpdatePeriod - delay).Seconds()) > 31 { - t.Fatalf("update 2, delay jitter was more than 30 seconds") - } - gotDB = &database{last: db.last, tfu: db.tfu, tfl: db.tfl} - wantDB.last = now - if !reflect.DeepEqual(gotDB.tfu, wantDB.tfu) { - t.Errorf("update 2, threats for update mismatch:\ngot %+v\nwant %+v", gotDB.tfu, wantDB.tfu) - } - if !reflect.DeepEqual(gotDB.tfl, wantDB.tfl) { - t.Fatalf("update 2, threats for lookup mismatch:\ngot %+v\nwant %+v", gotDB.tfl, wantDB.tfl) - } - - // Update 3: full update and partial update with removals and additions. - now = now.Add(time.Hour) - resp = pb.FetchThreatListUpdatesResponse{ - ListUpdateResponses: []*pb.FetchThreatListUpdatesResponse_ListUpdateResponse{ - newResp(td013, partial, []int32{0, 2, 4}, nil, - "a2", "bd3291e0b4fc7522ee3363376ded7801b316184722d83d224948889dfcf12465"), - newResp(td113, partial, nil, []string{"aaaa", "bbbb", "cccc"}, - "b2", "11c85195ae99540ac07f80e2905e6e39aaefc4ac94cd380f366e79ba83560566"), - newResp(td014, partial, []int32{1, 3}, []string{"eeee", "ffff"}, - "c2", "d8b19bc1d972cae450d65494565d2c04653894618faf9b37148e2c78ea3807e5"), - newResp(td114, full, nil, []string{"AAAA", "0421E"}, - "d2", "b742965b7a759ba0254685bfc6edae3b1ba54d0168fb86f526d6c79c3d44c753"), - }, - } - delay, updated = db.Update(context.Background(), mockAPI) - if db.err != nil || !updated { - t.Fatalf("update 3, unexpected update error: %v", db.err) - } - if math.Abs((config.UpdatePeriod - delay).Seconds()) > 31 { - t.Fatalf("update 3, delay jitter was more than 30 seconds") - } - gotDB = &database{last: db.last, tfu: db.tfu, tfl: db.tfl} - wantDB = &database{ - last: now, - tfu: threatsForUpdate{ - td013: {SHA256: gotDB.tfu[td013].SHA256, State: []byte{0x61, 0x32}}, - td113: {SHA256: gotDB.tfu[td113].SHA256, State: []byte{0x62, 0x32}}, - td014: {SHA256: gotDB.tfu[td014].SHA256, State: []byte{0x63, 0x32}}, - td114: {SHA256: gotDB.tfu[td114].SHA256, State: []byte{0x64, 0x32}}, - }, - tfl: threatsForLookup{ - td013: newHashSet([]hashPrefix{"0421f", "aaaa", "cccc"}), - td113: newHashSet([]hashPrefix{"aaaa", "bbbb", "cccc"}), - td014: newHashSet([]hashPrefix{"aaaa", "cccc", "eeee", "ffff"}), - td114: newHashSet([]hashPrefix{"0421E", "AAAA"}), - }, - } - if !reflect.DeepEqual(gotDB.tfu, wantDB.tfu) { - t.Errorf("update 3, threats for update mismatch:\ngot %+v\nwant %+v", gotDB.tfu, wantDB.tfu) - } - if !reflect.DeepEqual(gotDB.tfl, wantDB.tfl) { - t.Fatalf("update 3, threats for lookup mismatch:\ngot %+v\nwant %+v", gotDB.tfl, wantDB.tfl) - } - - // Update 4: invalid SHA256 checksum. - now = now.Add(time.Hour) - resp = pb.FetchThreatListUpdatesResponse{ - ListUpdateResponses: []*pb.FetchThreatListUpdatesResponse_ListUpdateResponse{ - newResp(td013, partial, []int32{0, 1}, []string{"fizz", "buzz"}, - "a3", "bad0bad0bad0bad0bad0bad0bad0bad0bad0bad0bad0bad0bad0bad0bad0bad0"), - }, - } - delay, updated = db.Update(context.Background(), mockAPI) - if db.err == nil || updated { - t.Fatalf("update 4, unexpected update success") - } - if math.Abs((config.UpdatePeriod - delay).Seconds()) > 31 { - t.Fatalf("update 4, delay jitter was more than 30 seconds") - } - gotDB = &database{last: db.last, tfu: db.tfu, tfl: db.tfl} - wantDB = &database{} - if !reflect.DeepEqual(gotDB, wantDB) { - t.Fatalf("update 4, database state mismatch:\ngot %+v\nwant %+v", gotDB, wantDB) - } - - // Update 5: removal index is out-of-bounds. - now = now.Add(time.Hour) - resp = pb.FetchThreatListUpdatesResponse{ - ListUpdateResponses: []*pb.FetchThreatListUpdatesResponse_ListUpdateResponse{ - newResp(td013, partial, []int32{9000}, []string{"fizz", "buzz"}, - "a4", "5d6506974928a003d2a0ccbd7a40b5341ad10578fd3f54527087c5ecbbd17a12"), - }, - } - delay, updated = db.Update(context.Background(), mockAPI) - if db.err == nil || updated { - t.Fatalf("update 5, unexpected update success") - } - if math.Abs((config.UpdatePeriod - delay).Seconds()) > 31 { - t.Fatalf("update 5, delay jitter was more than 30 seconds") - } - gotDB = &database{last: db.last, tfu: db.tfu, tfl: db.tfl} - wantDB = &database{} - if !reflect.DeepEqual(gotDB, wantDB) { - t.Fatalf("update 5, database state mismatch:\ngot %+v\nwant %+v", gotDB, wantDB) - } - - // Update 6: api is broken for some unknown reason. Checks the backoff - errResponse = errors.New("Something broke") - delay, updated = db.Update(context.Background(), mockAPI) - if db.err == nil || updated { - t.Fatalf("update 6, unexpected update success") - } - minDelay := baseRetryDelay.Seconds() * float64(1) * float64(1) - maxDelay := baseRetryDelay.Seconds() * float64(2) * float64(1) - if delay.Seconds() < minDelay || delay.Seconds() > maxDelay { - t.Fatalf("update 6, Expected delay %v to be between %v and %v", delay.Seconds(), minDelay, maxDelay) - } - - // Update 7: api is still broken, check backoff is larger - delay, updated = db.Update(context.Background(), mockAPI) - if db.err == nil || updated { - t.Fatalf("update 7, unexpected update success") - } - minDelay = baseRetryDelay.Seconds() * float64(1) * float64(2) - maxDelay = baseRetryDelay.Seconds() * float64(2) * float64(2) - if delay.Seconds() < minDelay || delay.Seconds() > maxDelay { - t.Fatalf("update 7, Expected delay %v to be between %v and %v", delay.Seconds(), minDelay, maxDelay) - } - - // Update 8: api is still broken, check that backoff is larger than before - delay, updated = db.Update(context.Background(), mockAPI) - if db.err == nil || updated { - t.Fatalf("update 8, unexpected update success") - } - minDelay = baseRetryDelay.Seconds() * float64(1) * float64(4) - maxDelay = baseRetryDelay.Seconds() * float64(2) * float64(4) - if delay.Seconds() < minDelay || delay.Seconds() > maxDelay { - t.Fatalf("update 8, Expected delay %v to be between %v and %v", delay.Seconds(), minDelay, maxDelay) - } -} - -func TestDatabaseLookup(t *testing.T) { - var ( - td000 = ThreatDescriptor{0, 0, 0} - td001 = ThreatDescriptor{0, 0, 1} - td012 = ThreatDescriptor{0, 1, 2} - td123 = ThreatDescriptor{1, 2, 3} - td234 = ThreatDescriptor{2, 3, 4} - td456 = ThreatDescriptor{4, 5, 6} - td567 = ThreatDescriptor{5, 6, 7} - td678 = ThreatDescriptor{6, 7, 8} - ) - - threatsEqual := func(a, b []ThreatDescriptor) bool { - ma := make(map[ThreatDescriptor]struct{}) - mb := make(map[ThreatDescriptor]struct{}) - for _, td := range a { - ma[td] = struct{}{} - } - for _, td := range b { - mb[td] = struct{}{} - } - return reflect.DeepEqual(ma, mb) - } - - db := &database{tfl: threatsForLookup{ - td000: newHashSet([]hashPrefix{ - "26e307", "524d", "5c6655d4"}), - td001: newHashSet([]hashPrefix{ - "26e307", "3f93", "5c6655d4", "7294"}), - td012: newHashSet([]hashPrefix{ - "3f93", "5c6655d3", "5c6655d4", "7294"}), - td123: newHashSet([]hashPrefix{ - "1e25395a9b1b8", "3f93", "5c6655d2", "5c6655d5", "7294", "cad78c1c"}), - td234: newHashSet([]hashPrefix{ - "1e25395a9b1b8", "cad78c628", "cad78c68"}), - td456: newHashSet([]hashPrefix{ - "524d", "59b8", "5c6655d3", "cad78c1c"}), - td567: newHashSet([]hashPrefix{ - "1e25395a9b1b8", "26e307", "524d", "5c6655d5", "7294", "cad78c1c"}), - td678: newHashSet([]hashPrefix{ - "1e25395a9b1b8", "524d", "524d", "7294", "cad78c628"}), - }} - - vectors := []struct { - input hashPrefix // Input full hash - output hashPrefix // Output partial hash - threats []ThreatDescriptor - }{{ - input: "3db40718dad209613a1fd9dced74dc0e", - output: "", // Not found - }, { - input: "59b8332112b29950f594cf957f4d0e63", - output: "59b8", - threats: []ThreatDescriptor{td456}, - }, { - input: "524dfa307ba397754a35dcce0ee5f54a", - output: "524d", - threats: []ThreatDescriptor{td000, td678, td456, td678, td567}, - }, { - input: "524dea307ba397754a35dcce0ee5f54a", - output: "524d", - threats: []ThreatDescriptor{td000, td678, td456, td678, td567}, - }, { - input: "5c6655d2096dd9ffb3c9e2bd5f86798f", - output: "5c6655d2", - threats: []ThreatDescriptor{td123}, - }, { - input: "5c6655d33db40718dad209613a1fd9dc", - output: "5c6655d3", - threats: []ThreatDescriptor{td012, td456}, - }, { - input: "1e25395a9b1b87db129a7d85ee7cc0fd", - output: "1e25395a9b1b8", - threats: []ThreatDescriptor{td123, td234, td567, td678}, - }} - - for i, v := range vectors { - ph, m := db.Lookup(v.input) - if ph != v.output { - t.Errorf("test %d, partial hash mismatch: got %s, want %s", i, ph, v.output) - } - if !threatsEqual(m, v.threats) { - t.Errorf("test %d, results mismatch: got %v, want %v", i, m, v.threats) - } - } -} - -func TestDatabasePersistence(t *testing.T) { - path := mustGetTempFile(t) - defer os.Remove(path) - - vectors := []struct { - last time.Time // Input last update time - tfu threatsForUpdate // Input threatsByDescriptor - }{{ - last: time.Time{}, - }, { - last: time.Now().Round(0), // Strip monotonic timestamp in Go1.9 - }, { - last: time.Unix(123456, 789), - tfu: threatsForUpdate{ - {0, 1, 2}: partialHashes{ - SHA256: mustDecodeHex(t, "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"), - }, - }, - }, { - last: time.Unix(987654321, 0), - tfu: threatsForUpdate{ - {3, 4, 5}: partialHashes{ - Hashes: []hashPrefix{"aaaa", "bbbb", "cccc", "dddd"}, - State: []byte("meow meow meow!!!"), - SHA256: mustDecodeHex(t, "147eb9dcde0e090429c01dbf634fd9b69a7f141f005c387a9c00498908499dde"), - }, - {7, 8, 9}: partialHashes{ - Hashes: []hashPrefix{"xxxx", "yyyy", "zzzz"}, - State: []byte("rawr rawr rawr!!!"), - SHA256: mustDecodeHex(t, "20ffb2c3e9532153b96b956845381adc06095f8342fa2db1aafba6b0e9594d68"), - }, - }, - }} - - for i, v := range vectors { - dbf1 := databaseFormat{v.tfu, v.last} - if err := saveDatabase(path, dbf1); err != nil { - t.Errorf("test %d, unexpected save error: %v", i, err) - continue - } - - dbf2, err := loadDatabase(path) - if err != nil { - t.Errorf("test %d, unexpected load error: %v", i, err) - continue - } - - if !reflect.DeepEqual(dbf1, dbf2) { - t.Errorf("test %d, mismatching database contents:\ngot %v\nwant %v", i, dbf2, dbf1) - } - } -} - -func TestDatabaseSaveErrors(t *testing.T) { - path := mustGetTempFile(t) - defer os.Remove(path) - - // Set mode to be unwritable. - if err := os.Chmod(path, 0444); err != nil { - t.Errorf("unexpected error: %v", err) - } - - if err := saveDatabase(path, databaseFormat{}); err == nil { - t.Errorf("unexpected save success") - } -} - -func TestDatabaseLoadErrors(t *testing.T) { - path := mustGetTempFile(t) - defer os.Remove(path) - - dbf1 := databaseFormat{ - Table: threatsForUpdate{ - {3, 4, 5}: partialHashes{ - Hashes: []hashPrefix{"aaaa", "bbbb", "cccc", "dddd"}, - State: []byte("meow meow meow!!!"), - SHA256: nil, // Intentionally leave this out - }, - }, - } - if err := saveDatabase(path, dbf1); err != nil { - t.Errorf("unexpected error: %v", err) - } - - if _, err := loadDatabase(path); err == nil { - t.Errorf("unexpected success") - } - - if err := os.Truncate(path, 13); err != nil { - t.Errorf("unexpected error: %v", err) - } - - if _, err := loadDatabase(path); err != io.ErrUnexpectedEOF { - t.Errorf("mismatching error: got %v, want %v", err, io.ErrUnexpectedEOF) - } -} - -func TestReady(t *testing.T) { - config := &Config{ - ThreatLists: []ThreatDescriptor{{0, 2, 3}}, - } - - db := new(database) - logger := log.New(ioutil.Discard, "", 0) - db.Init(config, logger) - // Expect timeout - ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond) - defer cancel() - select { - case <-db.Ready(): - t.Fatal("db.Ready() is closed, wanted timeout") - case <-ctx.Done(): - // expected - } - - done := make(chan bool) - go func(t *testing.T, done chan bool) { - ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond) - defer cancel() - select { - case <-db.Ready(): - // expected - case <-ctx.Done(): - t.Errorf("db.Ready() was not closed, expected close before timeout") - } - close(done) - }(t, done) - db.clearError() - <-done - -} - -func TestIsStale(t *testing.T) { - now := time.Unix(1451436338, 951473000) - mockNow := func() time.Time { return now } - db := new(database) - logger := log.New(ioutil.Discard, "", 0) - - config := &Config{ - UpdatePeriod: DefaultUpdatePeriod, - } - db.Init(config, logger) - db.config.now = mockNow - - vectors := []struct { - LastUpdate time.Time - ExpectedStale bool - }{ - // Last update of now isn't stale - {LastUpdate: now, ExpectedStale: false}, - // Last update between DefaultUpdatePeriod and 2*DefaultUpdatePeriod isn't stale - {LastUpdate: now.Add(-(DefaultUpdatePeriod + time.Minute)), ExpectedStale: false}, - // Last update right at the cusp of -2 * the DefaultUpdatePeriod isn't stale - {LastUpdate: now.Add(-2 * DefaultUpdatePeriod), ExpectedStale: false}, - // Last update right past -2 * DefaultUpdatePeriod + jitter is stale - {LastUpdate: now.Add(-2 * (DefaultUpdatePeriod + (2 * time.Minute))), ExpectedStale: true}, - // Last update well past -2 * DefaultUpdatePeriod + jitter is stale - {LastUpdate: now.Add(-3 * (DefaultUpdatePeriod + time.Minute)), ExpectedStale: true}, - } - - for i, v := range vectors { - stale := db.isStale(v.LastUpdate) - if stale != v.ExpectedStale { - t.Errorf("test %d, mismatching isStale: got %v, want %v", i, stale, v.ExpectedStale) - } - } -} diff --git a/go.mod b/go.mod index 50ac5c9..255916e 100644 --- a/go.mod +++ b/go.mod @@ -1,9 +1,16 @@ module github.com/google/safebrowsing +go 1.25.11 + require ( - github.com/golang/protobuf v1.2.0 + github.com/golang/protobuf v1.5.4 github.com/rakyll/statik v0.1.5 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd - golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 // indirect +) + +require ( + github.com/google/go-cmp v0.5.5 // indirect golang.org/x/text v0.3.0 // indirect + golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 // indirect + google.golang.org/protobuf v1.33.0 // indirect ) diff --git a/go.sum b/go.sum index 14079b3..ab889d7 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,8 @@ github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/rakyll/statik v0.1.5 h1:Ly2UjURzxnsSYS0zI50fZ+srA+Fu7EbpV5hglvJvJG0= github.com/rakyll/statik v0.1.5/go.mod h1:OEi9wJV/fMUAGx1eNjq75DKDsJVuEv1U0oYdX6GX8Zs= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd h1:HuTn7WObtcDo9uEEU7rEqL0jYthdXAmZ6PP+meazmaU= @@ -8,3 +11,6 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FY golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= diff --git a/hash.go b/hash.go index 430719d..729849e 100644 --- a/hash.go +++ b/hash.go @@ -17,12 +17,11 @@ package safebrowsing import ( "crypto/sha256" "encoding/binary" + "encoding/base64" "errors" "io" "sort" "strings" - - pb "github.com/google/safebrowsing/internal/safebrowsing_proto" ) const ( @@ -153,71 +152,25 @@ func (hs *hashSet) Lookup(h hashPrefix) int { return 0 } -// decodeHashes takes a ThreatEntrySet and returns a list of hashes that should -// be added to the local database. -func decodeHashes(input *pb.ThreatEntrySet) ([]hashPrefix, error) { - switch input.CompressionType { - case pb.CompressionType_RAW: - raw := input.GetRawHashes() - if raw == nil { - return nil, errors.New("safebrowsing: nil raw hashes") - } - if raw.PrefixSize < minHashPrefixLength || raw.PrefixSize > maxHashPrefixLength { - return nil, errors.New("safebrowsing: invalid hash prefix length") - } - if len(raw.RawHashes)%int(raw.PrefixSize) != 0 { - return nil, errors.New("safebrowsing: invalid raw hashes") - } - hashes := make([]hashPrefix, len(raw.RawHashes)/int(raw.PrefixSize)) - for i := range hashes { - hashes[i] = hashPrefix(raw.RawHashes[:raw.PrefixSize]) - raw.RawHashes = raw.RawHashes[raw.PrefixSize:] - } - return hashes, nil - case pb.CompressionType_RICE: - values, err := decodeRiceIntegers(input.GetRiceHashes()) - if err != nil { - return nil, err - } - hashes := make([]hashPrefix, 0, len(values)) - var buf [4]byte - for _, h := range values { - binary.LittleEndian.PutUint32(buf[:], h) - hashes = append(hashes, hashPrefix(buf[:])) - } - return hashes, nil - default: - return nil, errors.New("safebrowsing: invalid compression type") - } +func decodeBase64(s string) ([]byte, error) { + return base64.StdEncoding.DecodeString(s) } -// decodeIndices takes a ThreatEntrySet for removals returned by the server and -// returns a list of indices that the client should remove from its database. -func decodeIndices(input *pb.ThreatEntrySet) ([]int32, error) { - switch input.CompressionType { - case pb.CompressionType_RAW: - raw := input.GetRawIndices() - if raw == nil { - return nil, errors.New("safebrowsing: invalid raw indices") - } - return raw.Indices, nil - case pb.CompressionType_RICE: - values, err := decodeRiceIntegers(input.GetRiceIndices()) - if err != nil { - return nil, err - } - indices := make([]int32, 0, len(values)) - for _, v := range values { - indices = append(indices, int32(v)) - } - return indices, nil - default: - return nil, errors.New("safebrowsing: invalid compression type") +func decodeRice32Hashes(rice *rice32) ([]hashPrefix, error) { + values, err := decodeRice32(rice) + if err != nil { + return nil, err + } + hashes := make([]hashPrefix, 0, len(values)) + var buf [4]byte + for _, h := range values { + binary.BigEndian.PutUint32(buf[:], h) + hashes = append(hashes, hashPrefix(buf[:])) } + return hashes, nil } -// decodeRiceIntegers decodes a list of Golomb-Rice encoded integers. -func decodeRiceIntegers(rice *pb.RiceDeltaEncoding) ([]uint32, error) { +func decodeRice32(rice *rice32) ([]uint32, error) { if rice == nil { return nil, errors.New("safebrowsing: missing rice encoded data") } @@ -225,10 +178,14 @@ func decodeRiceIntegers(rice *pb.RiceDeltaEncoding) ([]uint32, error) { return nil, errors.New("safebrowsing: invalid k parameter") } - values := []uint32{uint32(rice.FirstValue)} - br := newBitReader(rice.EncodedData) + values := []uint32{rice.FirstValue} + encodedData, err := decodeBase64(rice.EncodedData) + if err != nil { + return nil, err + } + br := newBitReader(encodedData) rd := newRiceDecoder(br, uint32(rice.RiceParameter)) - for i := 0; i < int(rice.NumEntries); i++ { + for i := 0; i < int(rice.EntriesCount); i++ { delta, err := rd.ReadValue() if err != nil { return nil, err @@ -284,7 +241,7 @@ func (rd *riceDecoder) ReadValue() (uint32, error) { return 0, err } - return q<+ -// | |___________| | -// | | | -// | | Maybe? | -// | _____V_____ | -// V Yes | | No V -// +<----| API |---->+ -// | |___________| | -// V V -// (Yes, unsafe) (No, safe) -// -// Essentially the query is presented to three major components: The database, -// the cache, and the API. Each of these may satisfy the query immediately, -// or may say that it does not know and that the query should be satisfied by -// the next component. The goal of the database and cache is to satisfy as many -// queries as possible to avoid using the API. -// -// Starting with a user query, a hash of the query is performed to preserve -// privacy regarded the exact nature of the query. For example, if the query -// was for a URL, then this would be the SHA256 hash of the URL in question. -// -// Given a query hash, we first check the local database (which is periodically -// synced with the global Safe Browsing API servers). This database will either -// tell us that the query is definitely safe, or that it does not have -// enough information. -// -// If we are unsure about the query, we check the local cache, which can be used -// to satisfy queries immediately if the same query had been made recently. -// The cache will tell us that the query is either safe, unsafe, or unknown -// (because the it's not in the cache or the entry expired). -// -// If we are still unsure about the query, then we finally query the API server, -// which is guaranteed to return to us an authoritative answer, assuming no -// networking failures. -// -// For more information, see the API developer's guide: -// https://developers.google.com/safe-browsing/ +// Package safebrowsing implements a client for the Safe Browsing API v5 Local +// List Mode. It maintains a local hash-prefix database and only calls Google +// for full hashes after a local prefix hit. package safebrowsing import ( @@ -79,166 +19,76 @@ import ( "log" "sync/atomic" "time" - - pb "github.com/google/safebrowsing/internal/safebrowsing_proto" ) const ( - // DefaultServerURL is the default URL for the Safe Browsing API. - DefaultServerURL = "safebrowsing.googleapis.com" - - // DefaultUpdatePeriod is the default period for how often SafeBrowser will - // reload its blacklist database. - DefaultUpdatePeriod = 30 * time.Minute - - // DefaultID and DefaultVersion are the default client ID and Version - // strings to send with every API call. - DefaultID = "GoSafeBrowser" - DefaultVersion = "1.0.0" - - // DefaultRequestTimeout is the default amount of time a single - // api request can take. + DefaultServerURL = "safebrowsing.googleapis.com" + DefaultUpdatePeriod = 30 * time.Minute + DefaultID = "GoSafeBrowser" + DefaultVersion = "2.0.0" DefaultRequestTimeout = time.Minute ) -// Errors specific to this package. var ( errClosed = errors.New("safebrowsing: handler is closed") errStale = errors.New("safebrowsing: threat list is stale") ) -// ThreatType is an enumeration type for threats classes. Examples of threat -// classes are malware, social engineering, etc. -type ThreatType uint16 - -func (tt ThreatType) String() string { return pb.ThreatType(tt).String() } - -// List of ThreatType constants. -const ( - ThreatType_Malware = ThreatType(pb.ThreatType_MALWARE) - ThreatType_SocialEngineering = ThreatType(pb.ThreatType_SOCIAL_ENGINEERING) - ThreatType_UnwantedSoftware = ThreatType(pb.ThreatType_UNWANTED_SOFTWARE) - ThreatType_PotentiallyHarmfulApplication = ThreatType(pb.ThreatType_POTENTIALLY_HARMFUL_APPLICATION) -) - -// PlatformType is an enumeration type for platform classes. Examples of -// platform classes are Windows, Linux, Android, etc. -type PlatformType uint16 - -func (pt PlatformType) String() string { return pb.PlatformType(pt).String() } - -// List of PlatformType constants. -const ( - PlatformType_AnyPlatform = PlatformType(pb.PlatformType_ANY_PLATFORM) - PlatformType_AllPlatforms = PlatformType(pb.PlatformType_ALL_PLATFORMS) - - PlatformType_Windows = PlatformType(pb.PlatformType_WINDOWS) - PlatformType_Linux = PlatformType(pb.PlatformType_LINUX) - PlatformType_Android = PlatformType(pb.PlatformType_ANDROID) - PlatformType_OSX = PlatformType(pb.PlatformType_OSX) - PlatformType_iOS = PlatformType(pb.PlatformType_IOS) - PlatformType_Chrome = PlatformType(pb.PlatformType_CHROME) -) - -// ThreatEntryType is an enumeration type for threat entries. Examples of -// threat entries are via URLs, binary digests, and IP address ranges. -type ThreatEntryType uint16 - -func (tet ThreatEntryType) String() string { return pb.ThreatEntryType(tet).String() } +type ThreatType string -// List of ThreatEntryType constants. const ( - ThreatEntryType_URL = ThreatEntryType(pb.ThreatEntryType_URL) - - // These below are not supported yet. - ThreatEntryType_Executable = ThreatEntryType(pb.ThreatEntryType_EXECUTABLE) - ThreatEntryType_IPRange = ThreatEntryType(pb.ThreatEntryType_IP_RANGE) + ThreatTypeUnspecified ThreatType = "THREAT_TYPE_UNSPECIFIED" + ThreatTypeMalware ThreatType = "MALWARE" + ThreatTypeSocialEngineering ThreatType = "SOCIAL_ENGINEERING" + ThreatTypeUnwantedSoftware ThreatType = "UNWANTED_SOFTWARE" + ThreatTypePotentiallyHarmfulApplication ThreatType = "POTENTIALLY_HARMFUL_APPLICATION" ) -// DefaultThreatLists is the default list of threat lists that SafeBrowser -// will maintain. Do not modify this variable. -var DefaultThreatLists = []ThreatDescriptor{ - {ThreatType_Malware, PlatformType_AnyPlatform, ThreatEntryType_URL}, - {ThreatType_SocialEngineering, PlatformType_AnyPlatform, ThreatEntryType_URL}, - {ThreatType_UnwantedSoftware, PlatformType_AnyPlatform, ThreatEntryType_URL}, +func (tt ThreatType) String() string { + if tt == "" { + return string(ThreatTypeUnspecified) + } + return string(tt) } -// A ThreatDescriptor describes a given threat, which itself is composed of -// several parameters along different dimensions: ThreatType, PlatformType, and -// ThreatEntryType. -type ThreatDescriptor struct { - ThreatType ThreatType - PlatformType PlatformType - ThreatEntryType ThreatEntryType +type HashList struct { + Name string + ThreatType ThreatType } -// A URLThreat is a specialized ThreatDescriptor for the URL threat -// entry type. type URLThreat struct { Pattern string - ThreatDescriptor + HashList +} + +var DefaultHashLists = []HashList{ + {Name: "mw-4b", ThreatType: ThreatTypeMalware}, + {Name: "se-4b", ThreatType: ThreatTypeSocialEngineering}, + {Name: "uws-4b", ThreatType: ThreatTypeUnwantedSoftware}, } -// Config sets up the SafeBrowser object. type Config struct { - // ServerURL is the URL for the Safe Browsing API server. - // If empty, it defaults to DefaultServerURL. - ServerURL string - - // ProxyURL is the URL of the proxy to use for all requests. - // If empty, the underlying library uses $HTTP_PROXY environment variable. - ProxyURL string - - // APIKey is the key used to authenticate with the Safe Browsing API - // service. This field is required. - APIKey string - - // ID and Version are client metadata associated with each API request to - // identify the specific implementation of the client. - // They are similar in usage to the "User-Agent" in an HTTP request. - // If empty, these default to DefaultID and DefaultVersion, respectively. - ID string - Version string - - // DBPath is a path to a persistent database file. - // If empty, SafeBrowser operates in a non-persistent manner. - // This means that blacklist results will not be cached beyond the lifetime - // of the SafeBrowser object. - DBPath string - - // UpdatePeriod determines how often we update the internal list database. - // If zero value, it defaults to DefaultUpdatePeriod. - UpdatePeriod time.Duration - - // ThreatLists determines which threat lists that SafeBrowser should - // subscribe to. The threats reported by LookupURLs will only be ones that - // are specified by this list. - // If empty, it defaults to DefaultThreatLists. - ThreatLists []ThreatDescriptor - - // RequestTimeout determines the timeout value for the http client. + ServerURL string + ProxyURL string + APIKey string + ID string + Version string + DBPath string + UpdatePeriod time.Duration + HashLists []HashList RequestTimeout time.Duration - - // Logger is an io.Writer that allows SafeBrowser to write debug information - // intended for human consumption. - // If empty, no logs will be written. - Logger io.Writer - - // compressionTypes indicates how the threat entry sets can be compressed. - compressionTypes []pb.CompressionType + Logger io.Writer api api now func() time.Time } -// setDefaults configures Config to have default parameters. -// It reports whether the current configuration is valid. func (c *Config) setDefaults() bool { if c.ServerURL == "" { c.ServerURL = DefaultServerURL } - if len(c.ThreatLists) == 0 { - c.ThreatLists = DefaultThreatLists + if len(c.HashLists) == 0 { + c.HashLists = DefaultHashLists } if c.UpdatePeriod <= 0 { c.UpdatePeriod = DefaultUpdatePeriod @@ -246,63 +96,53 @@ func (c *Config) setDefaults() bool { if c.RequestTimeout <= 0 { c.RequestTimeout = DefaultRequestTimeout } - if c.compressionTypes == nil { - c.compressionTypes = []pb.CompressionType{pb.CompressionType_RAW, pb.CompressionType_RICE} + if c.ID == "" { + c.ID = DefaultID + } + if c.Version == "" { + c.Version = DefaultVersion } return true } func (c Config) copy() Config { c2 := c - c2.ThreatLists = append([]ThreatDescriptor(nil), c.ThreatLists...) - c2.compressionTypes = append([]pb.CompressionType(nil), c.compressionTypes...) + c2.HashLists = append([]HashList(nil), c.HashLists...) return c2 } -// SafeBrowser is a client implementation of API v4. -// -// It provides a set of lookup methods that allows the user to query whether -// certain entries are considered a threat. The implementation manages all of -// local database and caching that would normally be needed to interact -// with the API server. +func (c Config) userAgent() string { + return c.ID + "/" + c.Version +} + type SafeBrowser struct { - stats Stats // Must be first for 64-bit alignment on non 64-bit systems. + stats Stats config Config api api db database c cache - - lists map[ThreatDescriptor]bool - - log *log.Logger - + lists map[string]HashList + log *log.Logger closed uint32 - done chan bool // Signals that the updater routine should stop + done chan bool } -// Stats records statistics regarding SafeBrowser's operation. type Stats struct { - QueriesByDatabase int64 // Number of queries satisfied by the database alone - QueriesByCache int64 // Number of queries satisfied by the cache alone - QueriesByAPI int64 // Number of queries satisfied by an API call - QueriesFail int64 // Number of queries that could not be satisfied - DatabaseUpdateLag time.Duration // Duration since last *missed* update. 0 if next update is in the future. + QueriesByDatabase int64 + QueriesByCache int64 + QueriesByAPI int64 + QueriesFail int64 + DatabaseUpdateLag time.Duration } -// NewSafeBrowser creates a new SafeBrowser. -// -// The conf struct allows the user to configure many aspects of the -// SafeBrowser's operation. func NewSafeBrowser(conf Config) (*SafeBrowser, error) { conf = conf.copy() if !conf.setDefaults() { return nil, errors.New("safebrowsing: invalid configuration") } - - // Create the SafeBrowsing object. if conf.api == nil { var err error - conf.api, err = newNetAPI(conf.ServerURL, conf.APIKey, conf.ProxyURL) + conf.api, err = newNetAPI(conf.ServerURL, conf.APIKey, conf.ProxyURL, conf.userAgent()) if err != nil { return nil, err } @@ -310,50 +150,37 @@ func NewSafeBrowser(conf Config) (*SafeBrowser, error) { if conf.now == nil { conf.now = time.Now } + sb := &SafeBrowser{ config: conf, api: conf.api, c: cache{now: conf.now}, + lists: make(map[string]HashList), } - - // TODO: Verify that config.ThreatLists is a subset of the list obtained - // by "/v4/threatLists" API endpoint. - - // Convert threat lists slice to a map for O(1) lookup. - sb.lists = make(map[ThreatDescriptor]bool) - for _, td := range conf.ThreatLists { - sb.lists[td] = true + for _, list := range conf.HashLists { + sb.lists[list.Name] = list } - // Setup the logger. w := conf.Logger - if conf.Logger == nil { + if w == nil { w = ioutil.Discard } sb.log = log.New(w, "safebrowsing: ", log.Ldate|log.Ltime|log.Lshortfile) delay := time.Duration(0) - // If database file is provided, use that to initialize. if !sb.db.Init(&sb.config, sb.log) { ctx, cancel := context.WithTimeout(context.Background(), sb.config.RequestTimeout) delay, _ = sb.db.Update(ctx, sb.api) cancel() } else { - if age := sb.db.SinceLastUpdate(); age < sb.config.UpdatePeriod { - delay = sb.config.UpdatePeriod - age - } + delay = sb.db.UntilNextUpdate() } - // Start the background list updater. sb.done = make(chan bool) go sb.updater(delay) return sb, nil } -// Status reports the status of SafeBrowser. It returns some statistics -// regarding the operation, and an error representing the status of its -// internal state. Most errors are transient and will recover themselves -// after some period. func (sb *SafeBrowser) Status() (Stats, error) { stats := Stats{ QueriesByDatabase: atomic.LoadInt64(&sb.stats.QueriesByDatabase), @@ -365,9 +192,10 @@ func (sb *SafeBrowser) Status() (Stats, error) { return stats, sb.db.Status() } -// WaitUntilReady blocks until the database is not in an error state. -// Returns nil when the database is ready. Returns an error if the provided -// context is canceled or if the SafeBrowser instance is Closed. +func (sb *SafeBrowser) DatabaseHealth() DatabaseHealth { + return sb.db.Health() +} + func (sb *SafeBrowser) WaitUntilReady(ctx context.Context) error { if atomic.LoadUint32(&sb.closed) == 1 { return errClosed @@ -382,36 +210,15 @@ func (sb *SafeBrowser) WaitUntilReady(ctx context.Context) error { } } -// LookupURLs looks up the provided URLs. It returns a list of threats, one for -// every URL requested, and an error if any occurred. It is safe to call this -// method concurrently. -// -// The outer dimension is across all URLs requested, and will always have the -// same length as urls regardless of whether an error occurs or not. -// The inner dimension is across every fragment that a given URL produces. -// For some URL at index i, one can check for a hit on any blacklist by -// checking if len(threats[i]) > 0. -// The ThreatEntryType field in the inner ThreatDescriptor will be set to -// ThreatEntryType_URL as this is a URL lookup. -// -// If an error occurs, the caller should treat the threats list returned as a -// best-effort response to the query. The results may be stale or be partial. -func (sb *SafeBrowser) LookupURLs(urls []string) (threats [][]URLThreat, err error) { - threats, err = sb.LookupURLsContext(context.Background(), urls) - return threats, err +func (sb *SafeBrowser) LookupURLs(urls []string) ([][]URLThreat, error) { + return sb.LookupURLsContext(context.Background(), urls) } -// LookupURLsContext looks up the provided URLs. The request will be canceled -// if the provided Context is canceled, or if Config.RequestTimeout has -// elapsed. It is safe to call this method concurrently. -// -// See LookupURLs for details on the returned results. func (sb *SafeBrowser) LookupURLsContext(ctx context.Context, urls []string) (threats [][]URLThreat, err error) { ctx, cancel := context.WithTimeout(ctx, sb.config.RequestTimeout) defer cancel() threats = make([][]URLThreat, len(urls)) - if atomic.LoadUint32(&sb.closed) != 0 { return threats, errClosed } @@ -423,135 +230,130 @@ func (sb *SafeBrowser) LookupURLsContext(ctx context.Context, urls []string) (th hashes := make(map[hashPrefix]string) hash2idxs := make(map[hashPrefix][]int) + prefixes := make(map[hashPrefix]struct{}) + hashListsByHash := make(map[hashPrefix][]HashList) - // Construct the follow-up request being made to the server. - // In the request, we only ask for partial hashes for privacy reasons. - req := &pb.FindFullHashesRequest{ - Client: &pb.ClientInfo{ - ClientId: sb.config.ID, - ClientVersion: sb.config.Version, - }, - ThreatInfo: &pb.ThreatInfo{}, - } - ttm := make(map[pb.ThreatType]bool) - ptm := make(map[pb.PlatformType]bool) - tetm := make(map[pb.ThreatEntryType]bool) - - for i, url := range urls { - urlhashes, err := generateHashes(url) + for i, rawURL := range urls { + urlHashes, err := generateHashes(rawURL) if err != nil { sb.log.Printf("error generating urlhashes: %v", err) atomic.AddInt64(&sb.stats.QueriesFail, int64(len(urls)-i)) return threats, err } - for fullHash, pattern := range urlhashes { + for fullHash, pattern := range urlHashes { hash2idxs[fullHash] = append(hash2idxs[fullHash], i) - _, alreadyRequested := hashes[fullHash] hashes[fullHash] = pattern - // Lookup in database according to threat list. - partialHash, unsureThreats := sb.db.Lookup(fullHash) - if len(unsureThreats) == 0 { - atomic.AddInt64(&sb.stats.QueriesByDatabase, 1) - continue // There are definitely no threats for this full hash - } - - // Lookup in cache according to recently seen values. cachedThreats, cr := sb.c.Lookup(fullHash) switch cr { case positiveCacheHit: - // The cache remembers this full hash as a threat. - // The threats we return to the client is the set intersection - // of unsureThreats and cachedThreats. - for _, td := range unsureThreats { - if _, ok := cachedThreats[td]; ok { - threats[i] = append(threats[i], URLThreat{ - Pattern: pattern, - ThreatDescriptor: td, - }) + for _, list := range sb.config.HashLists { + if cachedThreats[list.ThreatType] { + threats[i] = append(threats[i], URLThreat{Pattern: pattern, HashList: list}) } } atomic.AddInt64(&sb.stats.QueriesByCache, 1) + continue case negativeCacheHit: - // This is cached as a non-threat. atomic.AddInt64(&sb.stats.QueriesByCache, 1) continue - default: - // The cache knows nothing about this full hash, so we must make - // a request for it. - if alreadyRequested { - continue - } - for _, td := range unsureThreats { - ttm[pb.ThreatType(td.ThreatType)] = true - ptm[pb.PlatformType(td.PlatformType)] = true - tetm[pb.ThreatEntryType(td.ThreatEntryType)] = true - } - req.ThreatInfo.ThreatEntries = append(req.ThreatInfo.ThreatEntries, - &pb.ThreatEntry{Hash: []byte(partialHash)}) } + + partialHash, possibleLists := sb.db.Lookup(fullHash) + if len(possibleLists) == 0 { + atomic.AddInt64(&sb.stats.QueriesByDatabase, 1) + continue + } + hashListsByHash[fullHash] = possibleLists + prefixes[partialHash] = struct{}{} } } - for tt := range ttm { - req.ThreatInfo.ThreatTypes = append(req.ThreatInfo.ThreatTypes, tt) - } - for pt := range ptm { - req.ThreatInfo.PlatformTypes = append(req.ThreatInfo.PlatformTypes, pt) - } - for tet := range tetm { - req.ThreatInfo.ThreatEntryTypes = append(req.ThreatInfo.ThreatEntryTypes, tet) + + if len(prefixes) == 0 { + return threats, nil } - // Actually query the Safe Browsing API for exact full hash matches. - if len(req.ThreatInfo.ThreatEntries) != 0 { - resp, err := sb.api.HashLookup(ctx, req) + reqPrefixes := make([]hashPrefix, 0, len(prefixes)) + for prefix := range prefixes { + reqPrefixes = append(reqPrefixes, prefix) + } + var matches []fullHash + for start := 0; start < len(reqPrefixes); start += maxHashPrefixesPerRequest { + end := start + maxHashPrefixesPerRequest + if end > len(reqPrefixes) { + end = len(reqPrefixes) + } + batch := reqPrefixes[start:end] + resp, err := sb.api.HashLookup(ctx, batch) if err != nil { sb.log.Printf("HashLookup failure: %v", err) atomic.AddInt64(&sb.stats.QueriesFail, 1) return threats, err } + sb.c.Update(batch, resp) + matches = append(matches, resp.FullHashes...) + atomic.AddInt64(&sb.stats.QueriesByAPI, 1) + } - // Update the cache. - sb.c.Update(req, resp) - - // Pull the information the client cares about out of the response. - for _, tm := range resp.GetMatches() { - fullHash := hashPrefix(tm.GetThreat().Hash) - if !fullHash.IsFull() { + for _, match := range matches { + fullHashBytes, err := decodeBase64(match.FullHash) + if err != nil { + continue + } + fullHash := hashPrefix(fullHashBytes) + if !fullHash.IsFull() { + continue + } + pattern, ok := hashes[fullHash] + idxs, foundIdxs := hash2idxs[fullHash] + if !ok || !foundIdxs { + continue + } + possibleLists := hashListsByHash[fullHash] + for _, detail := range match.FullHashDetails { + if !enforceableThreatDetail(detail) { continue } - pattern, ok := hashes[fullHash] - idxs, findidx := hash2idxs[fullHash] - if findidx && ok { - td := ThreatDescriptor{ - ThreatType: ThreatType(tm.ThreatType), - PlatformType: PlatformType(tm.PlatformType), - ThreatEntryType: ThreatEntryType(tm.ThreatEntryType), - } - if !sb.lists[td] { + for _, list := range possibleLists { + if list.ThreatType != detail.ThreatType { continue } for _, idx := range idxs { - threats[idx] = append(threats[idx], URLThreat{ - Pattern: pattern, - ThreatDescriptor: td, - }) + threats[idx] = append(threats[idx], URLThreat{Pattern: pattern, HashList: list}) } } } - atomic.AddInt64(&sb.stats.QueriesByAPI, 1) } + return threats, nil } -// TODO: Add other types of lookup when available. -// func (sb *SafeBrowser) LookupBinaries(digests []string) (threats []BinaryThreat, err error) -// func (sb *SafeBrowser) LookupAddresses(addrs []string) (threats [][]AddressThreat, err error) +func validThreatType(tt ThreatType) bool { + switch tt { + case ThreatTypeMalware, ThreatTypeSocialEngineering, ThreatTypeUnwantedSoftware, ThreatTypePotentiallyHarmfulApplication: + return true + default: + return false + } +} + +func enforceableThreatDetail(detail fullHashDetail) bool { + if !validThreatType(detail.ThreatType) { + return false + } + for _, attribute := range detail.Attributes { + switch attribute { + case "CANARY", "FRAME_ONLY": + return false + default: + // Unknown attributes invalidate the entire detail for forward compatibility. + return false + } + } + return true +} -// updater is a blocking method that periodically updates the local database. -// This should be run as a separate goroutine and will be automatically stopped -// when sb.Close is called. func (sb *SafeBrowser) updater(delay time.Duration) { for { sb.log.Printf("Next update in %v", delay) @@ -564,15 +366,12 @@ func (sb *SafeBrowser) updater(delay time.Duration) { sb.c.Purge() } cancel() - case <-sb.done: return } } } -// Close cleans up all resources. -// This method must not be called concurrently with other lookup methods. func (sb *SafeBrowser) Close() error { if atomic.LoadUint32(&sb.closed) == 0 { atomic.StoreUint32(&sb.closed, 1) diff --git a/safebrowser_system_test.go b/safebrowser_system_test.go deleted file mode 100644 index dec415c..0000000 --- a/safebrowser_system_test.go +++ /dev/null @@ -1,203 +0,0 @@ -// Copyright 2016 Google Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package safebrowsing - -import ( - "context" - "flag" - "testing" - "time" - - pb "github.com/google/safebrowsing/internal/safebrowsing_proto" -) - -// The system tests below are non-deterministic and operate by performing -// network requests against the Safe Browsing API servers. Thus, in order to -// operate they need the user's API key. This can be specified using the -apikey -// command-line flag when running the tests. -var apiKeyFlag = flag.String("apikey", "", "specify your Safe Browsing API key") - -func TestNetworkAPIUpdate(t *testing.T) { - if *apiKeyFlag == "" { - t.Skip() - } - - nm, err := newNetAPI(DefaultServerURL, *apiKeyFlag, "") - if err != nil { - t.Errorf("unexpected error: %v", err) - } - - lists := []*pb.FetchThreatListUpdatesRequest_ListUpdateRequest{ - { - ThreatType: pb.ThreatType_POTENTIALLY_HARMFUL_APPLICATION, - PlatformType: pb.PlatformType_ANDROID, - ThreatEntryType: pb.ThreatEntryType_URL, - }} - req := &pb.FetchThreatListUpdatesRequest{ListUpdateRequests: lists} - - dat, err := nm.ListUpdate(context.Background(), req) - if err != nil { - t.Fatal(err) - } - if got, want := len(dat.GetListUpdateResponses()), len(lists); got != want { - t.Fatalf("len(responseList.GetListUpdateResponses()) = %d, want %d", got, want) - } - - for i, resp := range dat.GetListUpdateResponses() { - if resp.ThreatType != lists[i].ThreatType { - t.Errorf("ThreatType: got: %s, want: %s", resp.ThreatType, lists[i].ThreatType) - } - if resp.PlatformType != lists[i].PlatformType { - t.Errorf("PlatformType: got: %v, want: %s", resp.PlatformType, lists[i].PlatformType) - } - if resp.ThreatEntryType != lists[i].ThreatEntryType { - t.Errorf("ThreatEntryType: got: %s, want: %s", resp.ThreatEntryType, lists[i].ThreatEntryType) - } - if resp.ResponseType.String() != "FULL_UPDATE" { - t.Errorf("ResponseType: got: %s, want: FULL_UPDATE", resp.ResponseType) - } - if n := len(resp.GetRemovals()); n != 0 { - t.Errorf("len(resp.GetRemovals(): got: %v, want: 0", n) - } - if n := len(resp.GetAdditions()); n == 0 { - t.Errorf("len(resp.GetAdditions()), got: %v, want: >0", n) - } - if len(resp.NewClientState) == 0 { - t.Errorf("Unexpected empty state") - } - } - - for _, u := range dat.GetListUpdateResponses() { - for _, a := range u.GetAdditions() { - hashes, err := decodeHashes(a) - if err != nil { - t.Fatal(err) - } - for i := 0; i < len(hashes) && i < 10; i++ { - hash := hashes[i] - threats := []*pb.ThreatEntry{{Hash: []byte(hash)}} - fullHashReq := &pb.FindFullHashesRequest{ - ThreatInfo: &pb.ThreatInfo{ - PlatformTypes: []pb.PlatformType{u.PlatformType}, - ThreatTypes: []pb.ThreatType{u.ThreatType}, - ThreatEntryTypes: []pb.ThreatEntryType{u.ThreatEntryType}, - ThreatEntries: threats, - }, - } - fullHashResp, err := nm.HashLookup(context.Background(), fullHashReq) - if err != nil { - t.Fatal(err) - } - if got := len(fullHashResp.GetMatches()); got < 1 { - t.Fatalf("len(r.GetMatches()), got: %v, want: > 0", got) - } - } - } - } -} - -func TestNetworkAPILookup(t *testing.T) { - if *apiKeyFlag == "" { - t.Skip() - } - - nm, err := newNetAPI(DefaultServerURL, *apiKeyFlag, "") - if err != nil { - t.Errorf("unexpected error: %v", err) - } - - var c = pb.FetchThreatListUpdatesRequest_ListUpdateRequest{ - ThreatType: pb.ThreatType_POTENTIALLY_HARMFUL_APPLICATION, - PlatformType: pb.PlatformType_ANDROID, - ThreatEntryType: pb.ThreatEntryType_URL, - } - url := "testsafebrowsing.appspot.com/apiv4/" + c.PlatformType.String() + "/" + - c.ThreatType.String() + "/" + c.ThreatEntryType.String() + "/" - hash := hashFromPattern(url) - req := &pb.FindFullHashesRequest{ - ThreatInfo: &pb.ThreatInfo{ - ThreatTypes: []pb.ThreatType{c.ThreatType}, - PlatformTypes: []pb.PlatformType{c.PlatformType}, - ThreatEntryTypes: []pb.ThreatEntryType{c.ThreatEntryType}, - ThreatEntries: []*pb.ThreatEntry{{Hash: []byte(hash[:minHashPrefixLength])}}, - }, - } - resp, err := nm.HashLookup(context.Background(), req) - if err != nil { - t.Fatalf("Lookup failed: %v", err) - } - if len(resp.GetMatches()) < 1 { - t.Fatalf("No matches returned. Resp %v. Url %v.", resp.String(), url) - } -} - -func TestSafeBrowser(t *testing.T) { - if *apiKeyFlag == "" { - t.Skip() - } - - sb, err := NewSafeBrowser(Config{ - APIKey: *apiKeyFlag, - ID: "GoSafeBrowserSystemTest", - DBPath: "/tmp/safebrowser.db", - UpdatePeriod: 10 * time.Second, - ThreatLists: []ThreatDescriptor{ - {ThreatType_PotentiallyHarmfulApplication, PlatformType_Android, ThreatEntryType_URL}, - }, - }) - if err != nil { - t.Fatal(err) - } - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - if err := sb.WaitUntilReady(ctx); err != nil { - t.Fatal(err) - } - cancel() - - var c = pb.FetchThreatListUpdatesRequest_ListUpdateRequest{ - ThreatType: pb.ThreatType_POTENTIALLY_HARMFUL_APPLICATION, - PlatformType: pb.PlatformType_ANDROID, - ThreatEntryType: pb.ThreatEntryType_URL, - } - url := "http://testsafebrowsing.appspot.com/apiv4/" + c.PlatformType.String() + "/" + - c.ThreatType.String() + "/" + c.ThreatEntryType.String() + "/" - - urls := []string{url, url + "?q=test"} - threats, e := sb.LookupURLs(urls) - if e != nil { - t.Fatal(e) - } - if len(threats[0]) != 1 || len(threats[1]) != 1 { - t.Errorf("lookupURL failed") - } - - if err := sb.Close(); err != nil { - t.Fatal(err) - } - ctx, cancel = context.WithTimeout(context.Background(), time.Millisecond) - if err := sb.WaitUntilReady(ctx); err != errClosed { - t.Errorf("sb.WaitUntilReady() = %v on closed SafeBrowser, want %v", err, errClosed) - } - cancel() - - for _, hs := range sb.db.tfl { - if hs.Len() == 0 { - t.Errorf("Database length: got %d,, want >0", hs.Len()) - } - } - if len(sb.c.pttls) != 1 { - t.Errorf("Cache length: got %d, want 1", len(sb.c.pttls)) - } -} diff --git a/v5_test.go b/v5_test.go new file mode 100644 index 0000000..1e96012 --- /dev/null +++ b/v5_test.go @@ -0,0 +1,214 @@ +package safebrowsing + +import ( + "context" + "encoding/base64" + "encoding/hex" + "errors" + "log" + "reflect" + "testing" + "time" +) + +type mockAPI struct { + listUpdate func(context.Context, []string, map[string][]byte) (*hashListsResponse, error) + hashLookup func(context.Context, []hashPrefix) (*hashSearchResponse, error) +} + +func (m *mockAPI) ListUpdate(ctx context.Context, names []string, versions map[string][]byte) (*hashListsResponse, error) { + return m.listUpdate(ctx, names, versions) +} + +func (m *mockAPI) HashLookup(ctx context.Context, prefixes []hashPrefix) (*hashSearchResponse, error) { + return m.hashLookup(ctx, prefixes) +} + +func mustDecodeHex(t *testing.T, s string) []byte { + t.Helper() + buf, err := hex.DecodeString(s) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + return buf +} + +func newHashSet(phs hashPrefixes) (hs hashSet) { + hs.Import(phs) + return hs +} + +func TestDecodeRice32HashesUsesBigEndian(t *testing.T) { + hashes, err := decodeRice32Hashes(&rice32{FirstValue: 0x01020304}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got, want := []byte(hashes[0]), []byte{0x01, 0x02, 0x03, 0x04}; !reflect.DeepEqual(got, want) { + t.Fatalf("hash mismatch: got %x, want %x", got, want) + } +} + +func TestCacheUpdateAndLookup(t *testing.T) { + now := time.Unix(1000, 0) + c := cache{now: func() time.Time { return now }} + full := hashPrefix("aaaabbbbccccddddeeeeffffgggghhhh") + resp := &hashSearchResponse{ + CacheDuration: "60s", + FullHashes: []fullHash{{ + FullHash: base64.StdEncoding.EncodeToString([]byte(full)), + FullHashDetails: []fullHashDetail{{ThreatType: ThreatTypeMalware}}, + }}, + } + c.Update([]hashPrefix{"aaaa"}, resp) + + threats, result := c.Lookup(full) + if result != positiveCacheHit || !threats[ThreatTypeMalware] { + t.Fatalf("positive cache mismatch: got %v %v", threats, result) + } + _, result = c.Lookup(hashPrefix("aaaaccccddddeeeeffffgggghhhhzzzz")) + if result != negativeCacheHit { + t.Fatalf("negative cache mismatch: got %v", result) + } +} + +func TestCacheIgnoresNonEnforceableAndUnknownDetails(t *testing.T) { + now := time.Unix(1000, 0) + c := cache{now: func() time.Time { return now }} + full := hashPrefix("aaaabbbbccccddddeeeeffffgggghhhh") + resp := &hashSearchResponse{ + CacheDuration: "60s", + FullHashes: []fullHash{{ + FullHash: base64.StdEncoding.EncodeToString([]byte(full)), + FullHashDetails: []fullHashDetail{ + {ThreatType: ThreatTypeMalware, Attributes: []string{"CANARY"}}, + {ThreatType: ThreatTypeMalware, Attributes: []string{"FRAME_ONLY"}}, + {ThreatType: ThreatTypeMalware, Attributes: []string{"THREAT_ATTRIBUTE_UNSPECIFIED"}}, + }, + }}, + } + c.Update([]hashPrefix{"aaaa"}, resp) + + if _, result := c.Lookup(full); result != negativeCacheHit { + t.Fatalf("non-enforceable details were cached as threats: %v", result) + } +} + +func TestEnforceableThreatDetail(t *testing.T) { + tests := []struct { + name string + detail fullHashDetail + want bool + }{ + {name: "plain threat", detail: fullHashDetail{ThreatType: ThreatTypeMalware}, want: true}, + {name: "unknown threat", detail: fullHashDetail{ThreatType: ThreatTypeUnspecified}, want: false}, + {name: "canary", detail: fullHashDetail{ThreatType: ThreatTypeMalware, Attributes: []string{"CANARY"}}, want: false}, + {name: "frame only", detail: fullHashDetail{ThreatType: ThreatTypeMalware, Attributes: []string{"FRAME_ONLY"}}, want: false}, + {name: "unknown attribute", detail: fullHashDetail{ThreatType: ThreatTypeMalware, Attributes: []string{"NEW_ATTRIBUTE"}}, want: false}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := enforceableThreatDetail(test.detail); got != test.want { + t.Fatalf("got %v, want %v", got, test.want) + } + }) + } +} + +func TestDatabaseUpdate(t *testing.T) { + now := time.Unix(1000, 0) + config := &Config{ + HashLists: []HashList{{Name: "mw-4b", ThreatType: ThreatTypeMalware}}, + UpdatePeriod: time.Hour, + now: func() time.Time { return now }, + } + db := &database{config: config, log: log.New(testWriter{}, "", 0)} + resp := &hashListsResponse{HashLists: []hashList{{ + Name: "mw-4b", + Version: base64.StdEncoding.EncodeToString([]byte("v1")), + MinimumWaitDuration: "20m", + SHA256Checksum: base64.StdEncoding.EncodeToString(mustDecodeHex(t, "61be55a8e2f6b4e172338bddf184d6dbee29c98853e0a0485ecee7f27b9af0b4")), + AdditionsFourBytes: &rice32{FirstValue: 0x61616161}, + }}} + + delay, ok := db.Update(context.Background(), &mockAPI{ + listUpdate: func(context.Context, []string, map[string][]byte) (*hashListsResponse, error) { + return resp, nil + }, + }) + if !ok || delay != 20*time.Minute { + t.Fatalf("unexpected update result: ok=%v delay=%v err=%v", ok, delay, db.err) + } + got := db.tfl["mw-4b"] + want := newHashSet([]hashPrefix{"aaaa"}) + if !reflect.DeepEqual(got, want) { + t.Fatalf("lookup table mismatch: got %+v, want %+v", got, want) + } + health := db.Health() + if !health.Healthy || health.LastUpdate != now || !health.NextUpdate.After(now) || !health.FreshUntil.After(health.NextUpdate) { + t.Fatalf("unexpected health after update: %+v", health) + } +} + +func TestDatabaseBecomesStaleAfterFreshnessDeadline(t *testing.T) { + now := time.Unix(1000, 0) + config := &Config{ + UpdatePeriod: time.Minute, + now: func() time.Time { return now }, + } + db := &database{ + config: config, + log: log.New(testWriter{}, "", 0), + readyCh: make(chan struct{}), + last: now.Add(-2 * time.Minute), + next: now.Add(-time.Minute), + freshUntil: now.Add(time.Second), + } + + if health := db.Health(); !health.Healthy { + t.Fatalf("database became stale too early: %+v", health) + } + now = now.Add(2 * time.Second) + if health := db.Health(); health.Healthy || health.Error == "" { + t.Fatalf("database did not become stale: %+v", health) + } +} + +func TestUpdateFailureKeepsFreshDatabaseAvailable(t *testing.T) { + now := time.Unix(1000, 0) + config := &Config{ + HashLists: []HashList{{Name: "mw-4b", ThreatType: ThreatTypeMalware}}, + UpdatePeriod: time.Minute, + now: func() time.Time { return now }, + } + db := &database{ + config: config, + log: log.New(testWriter{}, "", 0), + readyCh: make(chan struct{}), + tfl: threatsForLookup{"mw-4b": newHashSet([]hashPrefix{"aaaa"})}, + last: now.Add(-time.Minute), + next: now, + freshUntil: now.Add(time.Minute), + } + + _, ok := db.Update(context.Background(), &mockAPI{ + listUpdate: func(context.Context, []string, map[string][]byte) (*hashListsResponse, error) { + return nil, errors.New("temporary failure") + }, + }) + if ok { + t.Fatal("unexpected update success") + } + health := db.Health() + if !health.Healthy || health.LastUpdateError != "temporary failure" { + t.Fatalf("fresh database should remain available after transient failure: %+v", health) + } + if _, lists := db.Lookup(hashPrefix("aaaabbbbccccddddeeeeffffgggghhhh")); len(lists) != 1 { + t.Fatalf("lookup database was discarded after transient failure: %v", lists) + } +} + +type testWriter struct{} + +func (testWriter) Write(p []byte) (int, error) { + return len(p), nil +}