Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# Copyright 2024 The Sigstore Authors
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
15"""Machinery for signing and verification of ML models.
17The serialization API produces a manifest representation of the models, and we
18use that to implement integrity checking of models in different computational
19patterns. This means that all manifests need to be kept only in memory.
21Since we need to support multiple signing methods (e.g., Sigstore, key,
22certificate, etc.) , we provide a `Signer` abstract class with a single `sign`
23method that takes a signing payload and converts it to a signature in the
24supported format.
26Finally, every signature needs to be verified. We pair every `Signer` subclass
27with a `Verifier` which takes a signature, verify the authenticity of the
28payload and then expand that to a manifest.
30Regarding the data formats, for signing, we need to convert the manifest to the
31signing payload. We only support manifests serialized to in-toto formats
32described by https://github.com/in-toto/attestation/tree/main/spec/v1. The
33envelope format is DSSE, as described in
34https://github.com/secure-systems-lab/dsse. The signature is in a Sigstore
35bundle format over the DSSE payload. The format is described at
36https://docs.sigstore.dev/about/bundle/.
37"""
39import abc
40import json
41import pathlib
42from typing import Any, Self
44from in_toto_attestation.v1 import statement
46from model_signing import manifest
47from model_signing._hashing import hashing
48from model_signing._hashing import memory
51# The expected in-toto payload type for the signature.
52_IN_TOTO_JSON_PAYLOAD_TYPE: str = "application/vnd.in-toto+json"
55# The expected in-toto statement type for the signature.
56_IN_TOTO_STATEMENT_TYPE: str = "https://in-toto.io/Statement/v1"
59# The expected model signature predicate type.
60_PREDICATE_TYPE: str = "https://model_signing/signature/v1.0"
63# The expected model signature predicate type for v0.2 (compat)
64_PREDICATE_TYPE_COMPAT: str = "https://model_signing/Digests/v0.1"
67def dsse_payload_to_manifest(dsse_payload: dict[str, Any]) -> manifest.Manifest:
68 """Builds a manifest from the DSSE payload read from a signature.
70 The payload here is a dictionary that represents the payload part of the
71 DSSE envelope contained in the Sigstore bundle.
73 Args:
74 dsse_payload: The in-toto DSSE envelope to convert to manifest.
76 Returns:
77 A manifest representing the signed model.
79 Raises:
80 ValueError: The payload cannot be deserialized to a manifest.
81 """
82 obtained_predicate_type = dsse_payload["predicateType"]
83 if obtained_predicate_type != _PREDICATE_TYPE:
84 if obtained_predicate_type == _PREDICATE_TYPE_COMPAT:
85 return dsse_payload_to_manifest_compat(dsse_payload)
86 raise ValueError(
87 f"Predicate type mismatch, expected {_PREDICATE_TYPE}, "
88 f"got {obtained_predicate_type}"
89 )
91 subjects = dsse_payload["subject"]
92 if len(subjects) != 1:
93 raise ValueError(f"Expected only one subject, got {subjects}")
95 model_name = subjects[0]["name"]
96 expected_digest = subjects[0]["digest"]["sha256"]
98 predicate = dsse_payload["predicate"]
99 resources = predicate["resources"]
100 if not resources:
101 raise ValueError("Bundle contains no resources")
103 serialization_args = predicate["serialization"]
104 serialization = manifest.SerializationType.from_args(serialization_args)
106 hasher = memory.SHA256()
107 items = []
108 for resource in resources:
109 name = resource["name"]
110 algorithm = resource["algorithm"]
111 digest_value = resource["digest"]
112 digest = hashing.Digest(algorithm, bytes.fromhex(digest_value))
113 hasher.update(digest.digest_value)
114 items.append(serialization.new_item(name, digest))
116 obtained_digest = hasher.compute().digest_hex
117 if obtained_digest != expected_digest:
118 raise ValueError(
119 f"Manifest is inconsistent. Root digest is {expected_digest}, "
120 f"but the included resources hash to {obtained_digest}"
121 )
123 return manifest.Manifest(model_name, items, serialization)
126def dsse_payload_to_manifest_compat(
127 dsse_payload: dict[str, Any],
128) -> manifest.Manifest:
129 """Builds a manifest from the DSSE payload read from a signature.
131 This is the same as `dsse_payload_to_manifest` but using a DSSE payload as
132 defined at v0.2 release. This was experimental but got used in production
133 before v1.0 so we need to patch support to it while verifiers migrate to the
134 forward compatible format defined by v1.0.
136 Args:
137 dsse_payload: The in-toto DSSE envelope to convert to manifest.
139 Returns:
140 A manifest representing the signed model.
142 Raises:
143 ValueError: The payload cannot be deserialized to a manifest.
144 """
145 # Model name is not defined, use a constant.
146 model_name = "compat-undefined-not-present"
148 # Serialization format is not present, build a fake one.
149 serialization = manifest.SerializationType.from_args(
150 {"method": "files", "hash_type": "sha256", "allow_symlinks": False}
151 )
153 # The only field with actual content is the subject.
154 items = []
155 for subject in dsse_payload["subject"]:
156 name = subject["name"]
157 algorithm = "sha256" # hardcoded, the only supported one
158 digest_value = subject["digest"][algorithm]
159 digest = hashing.Digest(algorithm, bytes.fromhex(digest_value))
160 items.append(serialization.new_item(name, digest))
162 # There is no verification that the manifest is missing items at this point.
163 return manifest.Manifest(model_name, items, serialization)
166class Payload:
167 """In-toto payload used to represent a model for signing.
169 This payload represents all the object (files, shards, etc.) of the model
170 paired with their hashes. It can be seen as a serialization of a manifest.
171 The hashes are all recorded under the predicate, given that for the subject
172 we are limited on what hashes we can use
173 (https://github.com/sigstore/sigstore-python/issues/1018). Each hash follows
174 the format of a ResourceDescriptor: is an object containing a name for the
175 object, the hashing algorithm, and the digest value. These are recorded in
176 the predicate, as part of the `"resources"` list.
178 The subject is a name for the model (taken from the model's directory) and a
179 global digest over all the computed digests. This is SHA256 computed over
180 all the digests, in the order they show up in the predicate (we canonicalize
181 this to be in alphabetical order). This digest can be used to refer to the
182 model from other metadata documents without having to carry the entire set
183 of resource descriptors around.
185 To ensure backwards compatibility, the predicate contains a
186 `"serialization"` section which describes the method used to serialize a
187 model to the manifest used to generate this payload. The section includes a
188 method name and a list of all relevant values needed to recompute the
189 serialization.
191 Future extensions to the model signature (e.g., incorporating model cards,
192 etc.) can be added as part of the predicate. For v1.0 of the predicate the
193 only supported fields in the predicate are `"serialization"` and
194 `"resources"`. Any other field should be ignored by verifiers adhering to
195 v1.0 version.
197 Example:
198 ```json
199 {
200 "_type": "https://in-toto.io/Statement/v1",
201 "subject": [
202 {
203 "name": "sample_model",
204 "digest": {
205 "sha256": "143cc6..."
206 }
207 }
208 ],
209 "predicateType": "https://model_signing/signature/v1.0",
210 "predicate": {
211 "serialization": {
212 "method": "files",
213 "hash_type": "sha256",
214 "allow_symlinks": true
215 "ignore_paths": [
216 "model.sig",
217 ".git",
218 ".gitattributes",
219 ".github",
220 ".gitignore"
221 ],
222 },
223 "resources": [
224 {
225 "algorithm": "sha256",
226 "digest": "fdd892...",
227 "name": "d0/f00"
228 },
229 {
230 "algorithm": "sha256",
231 "digest": "e16940...",
232 "name": "d0/f01"
233 },
234 {
235 "algorithm": "sha256",
236 "digest": "407822...",
237 "name": "d0/f02"
238 },
239 ...
240 {
241 "algorithm": "sha256",
242 "digest": "912bcf...",
243 "name": "f3"
244 }
245 ]
246 }
247 }
248 ```
249 """
251 def __init__(self, manifest: manifest.Manifest):
252 """Builds an instance of this in-toto payload.
254 Args:
255 manifest: the manifest to convert to signing payload.
256 """
257 hasher = memory.SHA256()
258 resources = []
259 for descriptor in manifest.resource_descriptors():
260 hasher.update(descriptor.digest.digest_value)
261 resources.append(
262 {
263 "name": descriptor.identifier,
264 "algorithm": descriptor.digest.algorithm,
265 "digest": descriptor.digest.digest_hex,
266 }
267 )
269 root_digest = {"sha256": hasher.compute().digest_hex}
270 subject = statement.ResourceDescriptor(
271 name=manifest.model_name, digest=root_digest
272 ).pb
274 predicate = {
275 "serialization": manifest.serialization_type,
276 "resources": resources,
277 # other properties can go here
278 }
280 self.statement = statement.Statement(
281 subjects=[subject],
282 predicate_type=_PREDICATE_TYPE,
283 predicate=predicate,
284 )
287class Signature(metaclass=abc.ABCMeta):
288 """Signature class, wrapping a sigstore bundle.
290 We only support sigstore bundle signature formats, but we need to have two
291 separate classes for this given the need to support traditional signing as
292 well as Sigstore one. One class wraps around `sigstore_models.Bundle` and
293 the other around the bundle as defined by `sigstore_protobuf_specs`.
294 """
296 @abc.abstractmethod
297 def write(self, path: pathlib.Path) -> None:
298 """Writes the signature to disk, to the given path.
300 Args:
301 path: The path to write the signature to.
302 """
304 @abc.abstractmethod
305 def to_bytes(self) -> bytes:
306 """Serializes the signature to bytes.
308 Returns the same Sigstore bundle content that `write` persists to
309 disk, encoded as UTF-8 JSON. This lets callers in serverless or
310 pipeline contexts obtain the signature in memory without touching the
311 filesystem.
313 Returns:
314 The serialized signature, as UTF-8 encoded bytes.
315 """
317 @classmethod
318 @abc.abstractmethod
319 def read(cls, path: pathlib.Path) -> Self:
320 """Reads the signature from disk.
322 Does not perform any signature verification, except what is needed to
323 parse the signature file.
325 Args:
326 path: The path to read the signature from.
328 Returns:
329 An instance of the class which can be passed to a `Verifier` for
330 signature and integrity verification.
331 """
334class Signer(metaclass=abc.ABCMeta):
335 """Generic signer.
337 Each signer may implement its own mechanism for managing the key material.
338 """
340 @abc.abstractmethod
341 def sign(self, payload: Payload) -> Signature:
342 """Signs the provided signing payload.
344 Args:
345 payload: The `Payload` instance that should be signed.
347 Returns:
348 A valid signature.
349 """
352class Verifier(metaclass=abc.ABCMeta):
353 """Generic signature verifier.
355 Every subclass of `Verifier` is paired with a subclass of `Signer`. This is
356 to ensure that they support the same signature formats as well as have
357 similar key materials.
359 If the signature is valid, the payload is expanded to a `Manifest` instance
360 which can then be used to check the model integrity.
361 """
363 def verify(self, signature: Signature) -> manifest.Manifest:
364 """Verifies the signature.
366 Args:
367 signature: The signature to verify.
369 Returns:
370 A `manifest.Manifest` instance that represents the model.
372 Raises:
373 ValueError: Signature verification fails.
374 """
375 payload_type, payload = self._verify_signed_content(signature)
377 if payload_type != _IN_TOTO_JSON_PAYLOAD_TYPE:
378 raise ValueError(
379 f"Expected DSSE payload {_IN_TOTO_JSON_PAYLOAD_TYPE}, "
380 f"but got {payload_type}"
381 )
383 payload = json.loads(payload)
385 if payload["_type"] != _IN_TOTO_STATEMENT_TYPE:
386 raise ValueError(
387 f"Expected in-toto {_IN_TOTO_STATEMENT_TYPE} payload, "
388 f"but got {payload['_type']}"
389 )
391 return dsse_payload_to_manifest(payload)
393 @abc.abstractmethod
394 def _verify_signed_content(self, signature: Signature) -> tuple[str, bytes]:
395 """Verifies the signed content and extract payload type and payload.
397 Subclasses only need to implement this method.
399 Args:
400 signature: The signature to verify.
402 Returns:
403 A tuple containing the payload type and the payload (as a JSON
404 object loaded to a dictionary).
405 """