V5 changes

api.go

@@ -5,51 +5,83 @@
 // You may obtain a copy of the License at
 //
 // http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
 
 package safebrowsing
 
 import (
-	"bytes"
 	"context"
+	"encoding/base64"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"net/url"
 	"strings"
-
-	pb "github.com/google/safebrowsing/internal/safebrowsing_proto"
+	"time"
 
 	"github.com/golang/protobuf/proto"
 )
 
 const (
-	findHashPath    = "/v4/fullHashes:find"
-	fetchUpdatePath = "/v4/threatListUpdates:fetch"
+	batchGetPath = "/v5/hashLists:batchGet"
+	hashSearchPath = "/v5/hashes:search"
 )
 
-// The api interface specifies wrappers around the Safe Browsing API.
+// The api interface specifies wrappers around the Safe Browsing API v5.
 type api interface {
-	ListUpdate(ctx context.Context, req *pb.FetchThreatListUpdatesRequest) (*pb.FetchThreatListUpdatesResponse, error)
-	HashLookup(ctx context.Context, req *pb.FindFullHashesRequest) (*pb.FindFullHashesResponse, error)
+	ListUpdate(ctx context.Context, names []string, versions map[string][]byte) (*hashListsResponse, error)
+	HashLookup(ctx context.Context, prefixes []hashPrefix) (*hashSearchResponse, error)
 }
 
-// netAPI is an api object that talks to the server over HTTP.
 type netAPI struct {
 	client *http.Client
 	url    *url.URL
+	key    string
+	ua     string
+}
+
+type hashListsResponse struct {
+	HashLists []hashList `json:"hashLists"`
+}
+
+type hashList struct {
+	Name                string       `json:"name"`
+	Version             string       `json:"version"`
+	PartialUpdate       bool         `json:"partialUpdate"`
+	CompressedRemovals  *rice32      `json:"compressedRemovals"`
+	MinimumWaitDuration string       `json:"minimumWaitDuration"`
+	SHA256Checksum      string       `json:"sha256Checksum"`
+	AdditionsFourBytes  *rice32      `json:"additionsFourBytes"`
+	Metadata            hashMetadata `json:"metadata"`
+}
+
+type hashMetadata struct {
+	ThreatTypes []ThreatType `json:"threatTypes"`
+	HashLength  string       `json:"hashLength"`
+}
+
+type rice32 struct {
+	FirstValue    uint32 `json:"firstValue"`
+	RiceParameter int32  `json:"riceParameter"`
+	EntriesCount  int32  `json:"entriesCount"`
+	EncodedData   string `json:"encodedData"`
+}
+
+type hashSearchResponse struct {
+	FullHashes    []fullHash `json:"fullHashes"`
+	CacheDuration string     `json:"cacheDuration"`
+}
+
+type fullHash struct {
+	FullHash        string           `json:"fullHash"`
+	FullHashDetails []fullHashDetail `json:"fullHashDetails"`
+}
+
+type fullHashDetail struct {
+	ThreatType ThreatType `json:"threatType"`
+	Attributes []string   `json:"attributes"`
 }
 
-// newNetAPI creates a new netAPI object pointed at the provided root URL.
-// For every request, it will use the provided API key.
-// If a proxy URL is given, it will be used in place of the default $HTTP_PROXY.
-// If the protocol is not specified in root, then this defaults to using HTTPS.
-func newNetAPI(root string, key string, proxy string) (*netAPI, error) {
+func newNetAPI(root string, key string, proxy string, userAgent string) (*netAPI, error) {
 	if !strings.Contains(root, "://") {
 		root = "https://" + root
 	}
@@ -59,59 +91,142 @@ func newNetAPI(root string, key string, proxy string) (*netAPI, error) {
 	}
 
 	httpClient := &http.Client{}
-
 	if proxy != "" {
-		proxyUrl, err := url.Parse(proxy)
+		proxyURL, err := url.Parse(proxy)
 		if err != nil {
 			return nil, err
 		}
-		httpClient = &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxyUrl)}}
+		httpClient = &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxyURL)}}
 	}
 
+	return &netAPI{url: u, client: httpClient, key: key, ua: userAgent}, nil
+}
+
+func (a *netAPI) getProto(ctx context.Context, requestPath string, query url.Values, out proto.Message) error {
+	u := *a.url
+	u.Path = requestPath
 	q := u.Query()
-	q.Set("key", key)
-	q.Set("alt", "proto")
+	for k, vals := range query {
+		for _, v := range vals {
+			q.Add(k, v)
+		}
+	}
+	q.Set("key", a.key)
 	u.RawQuery = q.Encode()
-	return &netAPI{url: u, client: httpClient}, nil
-}
 
-// doRequests performs a POST to requestPath. It uses the marshaled form of req
-// as the request body payload, and automatically unmarshals the response body
-// payload as resp.
-func (a *netAPI) doRequest(ctx context.Context, requestPath string, req proto.Message, resp proto.Message) error {
-	p, err := proto.Marshal(req)
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
 	if err != nil {
 		return err
 	}
+	if a.ua != "" {
+		req.Header.Set("User-Agent", a.ua)
+	}
+	req.Header.Set("Accept", "application/x-protobuf")
 
-	u := *a.url // Make a copy of URL
-	u.Path = requestPath
-	httpReq, err := http.NewRequest("POST", u.String(), bytes.NewReader(p))
-	httpReq.Header.Add("Content-Type", "application/x-protobuf")
-	httpReq = httpReq.WithContext(ctx)
-	httpResp, err := a.client.Do(httpReq)
+	resp, err := a.client.Do(req)
 	if err != nil {
 		return err
 	}
-	defer httpResp.Body.Close()
-	if httpResp.StatusCode != 200 {
-		return fmt.Errorf("safebrowsing: unexpected server response code: %d", httpResp.StatusCode)
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
+		return fmt.Errorf("safebrowsing: unexpected server response code: %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))
 	}
-	body, err := ioutil.ReadAll(httpResp.Body)
+	body, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return err
 	}
-	return proto.Unmarshal(body, resp)
+	if err := proto.Unmarshal(body, out); err != nil {
+		return fmt.Errorf("safebrowsing: decode %s response (%s): %w", requestPath, resp.Header.Get("Content-Type"), err)
+	}
+	return nil
 }
 
-// ListUpdate issues a FetchThreatListUpdates API call and returns the response.
-func (a *netAPI) ListUpdate(ctx context.Context, req *pb.FetchThreatListUpdatesRequest) (*pb.FetchThreatListUpdatesResponse, error) {
-	resp := new(pb.FetchThreatListUpdatesResponse)
-	return resp, a.doRequest(ctx, fetchUpdatePath, req, resp)
+func (a *netAPI) ListUpdate(ctx context.Context, names []string, versions map[string][]byte) (*hashListsResponse, error) {
+	q := url.Values{}
+	for _, name := range names {
+		q.Add("names", name)
+		if version := versions[name]; len(version) > 0 {
+			q.Add("version", base64.StdEncoding.EncodeToString(version))
+		}
+	}
+
+	var wireResp batchGetHashListsResponse
+	if err := a.getProto(ctx, batchGetPath, q, &wireResp); err != nil {
+		return nil, err
+	}
+	resp := &hashListsResponse{HashLists: make([]hashList, 0, len(wireResp.HashLists))}
+	for _, wireList := range wireResp.HashLists {
+		list := hashList{
+			Name:           wireList.Name,
+			Version:        base64.StdEncoding.EncodeToString(wireList.Version),
+			PartialUpdate:  wireList.PartialUpdate,
+			SHA256Checksum: base64.StdEncoding.EncodeToString(wireList.SHA256Checksum),
+		}
+		if wait := wireList.MinimumWaitDuration; wait != nil {
+			list.MinimumWaitDuration = wait.Duration().String()
+		}
+		if removals := wireList.CompressedRemovals; removals != nil {
+			list.CompressedRemovals = rice32FromProto(removals)
+		}
+		if additions := wireList.AdditionsFourBytes; additions != nil {
+			list.AdditionsFourBytes = rice32FromProto(additions)
+		}
+		resp.HashLists = append(resp.HashLists, list)
+	}
+	return resp, nil
+}
+
+func (a *netAPI) HashLookup(ctx context.Context, prefixes []hashPrefix) (*hashSearchResponse, error) {
+	q := url.Values{}
+	for _, prefix := range prefixes {
+		q.Add("hashPrefixes", base64.StdEncoding.EncodeToString([]byte(prefix)))
+	}
+
+	var wireResp searchHashesResponse
+	if err := a.getProto(ctx, hashSearchPath, q, &wireResp); err != nil {
+		return nil, err
+	}
+	resp := &hashSearchResponse{FullHashes: make([]fullHash, 0, len(wireResp.FullHashes))}
+	if duration := wireResp.CacheDuration; duration != nil {
+		resp.CacheDuration = duration.Duration().String()
+	}
+	for _, wireHash := range wireResp.FullHashes {
+		match := fullHash{
+			FullHash:        base64.StdEncoding.EncodeToString(wireHash.FullHash),
+			FullHashDetails: make([]fullHashDetail, 0, len(wireHash.FullHashDetails)),
+		}
+		for _, wireDetail := range wireHash.FullHashDetails {
+			attributes := make([]string, 0, len(wireDetail.Attributes))
+			for _, attribute := range wireDetail.Attributes {
+				attributes = append(attributes, threatAttributeName(attribute))
+			}
+			match.FullHashDetails = append(match.FullHashDetails, fullHashDetail{
+				ThreatType: ThreatType(threatTypeName(wireDetail.ThreatType)),
+				Attributes: attributes,
+			})
+		}
+		resp.FullHashes = append(resp.FullHashes, match)
+	}
+	return resp, nil
 }
 
-// HashLookup issues a FindFullHashes API call and returns the response.
-func (a *netAPI) HashLookup(ctx context.Context, req *pb.FindFullHashesRequest) (*pb.FindFullHashesResponse, error) {
-	resp := new(pb.FindFullHashesResponse)
-	return resp, a.doRequest(ctx, findHashPath, req, resp)
+func rice32FromProto(value *riceDeltaEncoded32Bit) *rice32 {
+	return &rice32{
+		FirstValue:    value.FirstValue,
+		RiceParameter: value.RiceParameter,
+		EntriesCount:  value.EntriesCount,
+		EncodedData:   base64.StdEncoding.EncodeToString(value.EncodedData),
+	}
+}
+
+func parseDuration(s string) time.Duration {
+	if s == "" {
+		return 0
+	}
+	d, err := time.ParseDuration(s)
+	if err != nil {
+		return 0
+	}
+	return d
 }

api_wire.go

@@ -0,0 +1,106 @@
+package safebrowsing
+
+import (
+	"time"
+
+	"github.com/golang/protobuf/proto"
+)
+
+type batchGetHashListsResponse struct {
+	HashLists []*wireHashList `protobuf:"bytes,1,rep,name=hash_lists,json=hashLists,proto3"`
+}
+
+func (m *batchGetHashListsResponse) Reset()         { *m = batchGetHashListsResponse{} }
+func (m *batchGetHashListsResponse) String() string { return proto.CompactTextString(m) }
+func (*batchGetHashListsResponse) ProtoMessage()    {}
+
+type wireHashList struct {
+	Name                string                     `protobuf:"bytes,1,opt,name=name,proto3"`
+	Version             []byte                     `protobuf:"bytes,2,opt,name=version,proto3"`
+	PartialUpdate       bool                       `protobuf:"varint,3,opt,name=partial_update,json=partialUpdate,proto3"`
+	AdditionsFourBytes  *riceDeltaEncoded32Bit     `protobuf:"bytes,4,opt,name=additions_four_bytes,json=additionsFourBytes,proto3"`
+	CompressedRemovals  *riceDeltaEncoded32Bit     `protobuf:"bytes,5,opt,name=compressed_removals,json=compressedRemovals,proto3"`
+	MinimumWaitDuration *wireDuration              `protobuf:"bytes,6,opt,name=minimum_wait_duration,json=minimumWaitDuration,proto3"`
+	SHA256Checksum      []byte                     `protobuf:"bytes,7,opt,name=sha256_checksum,json=sha256Checksum,proto3"`
+}
+
+func (m *wireHashList) Reset()         { *m = wireHashList{} }
+func (m *wireHashList) String() string { return proto.CompactTextString(m) }
+func (*wireHashList) ProtoMessage()    {}
+
+type riceDeltaEncoded32Bit struct {
+	FirstValue    uint32 `protobuf:"varint,1,opt,name=first_value,json=firstValue,proto3"`
+	RiceParameter int32  `protobuf:"varint,2,opt,name=rice_parameter,json=riceParameter,proto3"`
+	EntriesCount  int32  `protobuf:"varint,3,opt,name=entries_count,json=entriesCount,proto3"`
+	EncodedData   []byte `protobuf:"bytes,4,opt,name=encoded_data,json=encodedData,proto3"`
+}
+
+func (m *riceDeltaEncoded32Bit) Reset()         { *m = riceDeltaEncoded32Bit{} }
+func (m *riceDeltaEncoded32Bit) String() string { return proto.CompactTextString(m) }
+func (*riceDeltaEncoded32Bit) ProtoMessage()    {}
+
+type wireDuration struct {
+	Seconds int64 `protobuf:"varint,1,opt,name=seconds,proto3"`
+	Nanos   int32 `protobuf:"varint,2,opt,name=nanos,proto3"`
+}
+
+func (m *wireDuration) Reset()         { *m = wireDuration{} }
+func (m *wireDuration) String() string { return proto.CompactTextString(m) }
+func (*wireDuration) ProtoMessage()    {}
+
+func (m *wireDuration) Duration() time.Duration {
+	return time.Duration(m.Seconds)*time.Second + time.Duration(m.Nanos)
+}
+
+type searchHashesResponse struct {
+	FullHashes    []*wireFullHash `protobuf:"bytes,1,rep,name=full_hashes,json=fullHashes,proto3"`
+	CacheDuration *wireDuration   `protobuf:"bytes,2,opt,name=cache_duration,json=cacheDuration,proto3"`
+}
+
+func (m *searchHashesResponse) Reset()         { *m = searchHashesResponse{} }
+func (m *searchHashesResponse) String() string { return proto.CompactTextString(m) }
+func (*searchHashesResponse) ProtoMessage()    {}
+
+type wireFullHash struct {
+	FullHash        []byte                `protobuf:"bytes,1,opt,name=full_hash,json=fullHash,proto3"`
+	FullHashDetails []*wireFullHashDetail `protobuf:"bytes,2,rep,name=full_hash_details,json=fullHashDetails,proto3"`
+}
+
+func (m *wireFullHash) Reset()         { *m = wireFullHash{} }
+func (m *wireFullHash) String() string { return proto.CompactTextString(m) }
+func (*wireFullHash) ProtoMessage()    {}
+
+type wireFullHashDetail struct {
+	ThreatType int32   `protobuf:"varint,1,opt,name=threat_type,json=threatType,proto3"`
+	Attributes []int32 `protobuf:"varint,2,rep,packed,name=attributes,proto3"`
+}
+
+func (m *wireFullHashDetail) Reset()         { *m = wireFullHashDetail{} }
+func (m *wireFullHashDetail) String() string { return proto.CompactTextString(m) }
+func (*wireFullHashDetail) ProtoMessage()    {}
+
+func threatTypeName(value int32) string {
+	switch value {
+	case 1:
+		return string(ThreatTypeMalware)
+	case 2:
+		return string(ThreatTypeSocialEngineering)
+	case 3:
+		return string(ThreatTypeUnwantedSoftware)
+	case 4:
+		return string(ThreatTypePotentiallyHarmfulApplication)
+	default:
+		return string(ThreatTypeUnspecified)
+	}
+}
+
+func threatAttributeName(value int32) string {
+	switch value {
+	case 1:
+		return "CANARY"
+	case 2:
+		return "FRAME_ONLY"
+	default:
+		return "THREAT_ATTRIBUTE_UNSPECIFIED"
+	}
+}