AI Newsletter Digest improvements: fixed QP soft line break decoding, URL extraction, and content cleaning

This commit is contained in:
Krilly
2026-03-04 13:29:22 +00:00
parent 29a98137a7
commit 57dd294675
13706 changed files with 2114953 additions and 237629 deletions

View File

@@ -0,0 +1,222 @@
# don't import any costly modules
import sys
import os
is_pypy = '__pypy__' in sys.builtin_module_names
def warn_distutils_present():
if 'distutils' not in sys.modules:
return
if is_pypy and sys.version_info < (3, 7):
# PyPy for 3.6 unconditionally imports distutils, so bypass the warning
# https://foss.heptapod.net/pypy/pypy/-/blob/be829135bc0d758997b3566062999ee8b23872b4/lib-python/3/site.py#L250
return
import warnings
warnings.warn(
"Distutils was imported before Setuptools, but importing Setuptools "
"also replaces the `distutils` module in `sys.modules`. This may lead "
"to undesirable behaviors or errors. To avoid these issues, avoid "
"using distutils directly, ensure that setuptools is installed in the "
"traditional way (e.g. not an editable install), and/or make sure "
"that setuptools is always imported before distutils."
)
def clear_distutils():
if 'distutils' not in sys.modules:
return
import warnings
warnings.warn("Setuptools is replacing distutils.")
mods = [
name
for name in sys.modules
if name == "distutils" or name.startswith("distutils.")
]
for name in mods:
del sys.modules[name]
def enabled():
"""
Allow selection of distutils by environment variable.
"""
which = os.environ.get('SETUPTOOLS_USE_DISTUTILS', 'local')
return which == 'local'
def ensure_local_distutils():
import importlib
clear_distutils()
# With the DistutilsMetaFinder in place,
# perform an import to cause distutils to be
# loaded from setuptools._distutils. Ref #2906.
with shim():
importlib.import_module('distutils')
# check that submodules load as expected
core = importlib.import_module('distutils.core')
assert '_distutils' in core.__file__, core.__file__
assert 'setuptools._distutils.log' not in sys.modules
def do_override():
"""
Ensure that the local copy of distutils is preferred over stdlib.
See https://github.com/pypa/setuptools/issues/417#issuecomment-392298401
for more motivation.
"""
if enabled():
warn_distutils_present()
ensure_local_distutils()
class _TrivialRe:
def __init__(self, *patterns):
self._patterns = patterns
def match(self, string):
return all(pat in string for pat in self._patterns)
class DistutilsMetaFinder:
def find_spec(self, fullname, path, target=None):
# optimization: only consider top level modules and those
# found in the CPython test suite.
if path is not None and not fullname.startswith('test.'):
return
method_name = 'spec_for_{fullname}'.format(**locals())
method = getattr(self, method_name, lambda: None)
return method()
def spec_for_distutils(self):
if self.is_cpython():
return
import importlib
import importlib.abc
import importlib.util
try:
mod = importlib.import_module('setuptools._distutils')
except Exception:
# There are a couple of cases where setuptools._distutils
# may not be present:
# - An older Setuptools without a local distutils is
# taking precedence. Ref #2957.
# - Path manipulation during sitecustomize removes
# setuptools from the path but only after the hook
# has been loaded. Ref #2980.
# In either case, fall back to stdlib behavior.
return
class DistutilsLoader(importlib.abc.Loader):
def create_module(self, spec):
mod.__name__ = 'distutils'
return mod
def exec_module(self, module):
pass
return importlib.util.spec_from_loader(
'distutils', DistutilsLoader(), origin=mod.__file__
)
@staticmethod
def is_cpython():
"""
Suppress supplying distutils for CPython (build and tests).
Ref #2965 and #3007.
"""
return os.path.isfile('pybuilddir.txt')
def spec_for_pip(self):
"""
Ensure stdlib distutils when running under pip.
See pypa/pip#8761 for rationale.
"""
if self.pip_imported_during_build():
return
clear_distutils()
self.spec_for_distutils = lambda: None
@classmethod
def pip_imported_during_build(cls):
"""
Detect if pip is being imported in a build script. Ref #2355.
"""
import traceback
return any(
cls.frame_file_is_setup(frame) for frame, line in traceback.walk_stack(None)
)
@staticmethod
def frame_file_is_setup(frame):
"""
Return True if the indicated frame suggests a setup.py file.
"""
# some frames may not have __file__ (#2940)
return frame.f_globals.get('__file__', '').endswith('setup.py')
def spec_for_sensitive_tests(self):
"""
Ensure stdlib distutils when running select tests under CPython.
python/cpython#91169
"""
clear_distutils()
self.spec_for_distutils = lambda: None
sensitive_tests = (
[
'test.test_distutils',
'test.test_peg_generator',
'test.test_importlib',
]
if sys.version_info < (3, 10)
else [
'test.test_distutils',
]
)
for name in DistutilsMetaFinder.sensitive_tests:
setattr(
DistutilsMetaFinder,
f'spec_for_{name}',
DistutilsMetaFinder.spec_for_sensitive_tests,
)
DISTUTILS_FINDER = DistutilsMetaFinder()
def add_shim():
DISTUTILS_FINDER in sys.meta_path or insert_shim()
class shim:
def __enter__(self):
insert_shim()
def __exit__(self, exc, value, tb):
remove_shim()
def insert_shim():
sys.meta_path.insert(0, DISTUTILS_FINDER)
def remove_shim():
try:
sys.meta_path.remove(DISTUTILS_FINDER)
except ValueError:
pass

View File

@@ -0,0 +1 @@
__import__('_distutils_hack').do_override()

View File

@@ -0,0 +1 @@
import os; var = 'SETUPTOOLS_USE_DISTUTILS'; enabled = os.environ.get(var, 'local') == 'local'; enabled and __import__('_distutils_hack').add_shim();

View File

@@ -0,0 +1,27 @@
Metadata-Version: 2.4
Name: flatbuffers
Version: 25.12.19
Summary: The FlatBuffers serialization format for Python
Home-page: https://google.github.io/flatbuffers/
Author: Derek Bailey
Author-email: derekbailey@google.com
License: Apache 2.0
Project-URL: Documentation, https://google.github.io/flatbuffers/
Project-URL: Source, https://github.com/google/flatbuffers
Classifier: Intended Audience :: Developers
Classifier: License :: OSI Approved :: Apache Software License
Classifier: Operating System :: OS Independent
Classifier: Programming Language :: Python
Classifier: Programming Language :: Python :: 2
Classifier: Programming Language :: Python :: 3
Classifier: Topic :: Software Development :: Libraries :: Python Modules
Dynamic: author
Dynamic: author-email
Dynamic: classifier
Dynamic: description
Dynamic: home-page
Dynamic: license
Dynamic: project-url
Dynamic: summary
Python runtime library for use with the `Flatbuffers <https://google.github.io/flatbuffers/>`_ serialization format.

View File

@@ -0,0 +1,25 @@
flatbuffers-25.12.19.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
flatbuffers-25.12.19.dist-info/METADATA,sha256=nzgjn5b6ohUipqratpEuHlYHCLvpyfaWoGeclAP6CJ4,1004
flatbuffers-25.12.19.dist-info/RECORD,,
flatbuffers-25.12.19.dist-info/WHEEL,sha256=JNWh1Fm1UdwIQV075glCn4MVuCRs0sotJIq-J6rbxCU,109
flatbuffers-25.12.19.dist-info/top_level.txt,sha256=UXVWLA8ys6HeqTz6rfKesocUq6ln-ZL8mhZC_cq5BEc,12
flatbuffers/__init__.py,sha256=vJZrqZOOTKdBNMa_iTKUA6WJG_c_NzKGpFXOe1Igtiw,751
flatbuffers/__pycache__/__init__.cpython-311.pyc,,
flatbuffers/__pycache__/_version.cpython-311.pyc,,
flatbuffers/__pycache__/builder.cpython-311.pyc,,
flatbuffers/__pycache__/compat.cpython-311.pyc,,
flatbuffers/__pycache__/encode.cpython-311.pyc,,
flatbuffers/__pycache__/flexbuffers.cpython-311.pyc,,
flatbuffers/__pycache__/number_types.cpython-311.pyc,,
flatbuffers/__pycache__/packer.cpython-311.pyc,,
flatbuffers/__pycache__/table.cpython-311.pyc,,
flatbuffers/__pycache__/util.cpython-311.pyc,,
flatbuffers/_version.py,sha256=sk31rbYWDseoJxI9YQ1fYR-MMxRATyqsntKAEv7D7pI,696
flatbuffers/builder.py,sha256=HrG5KJ9rasiSTrMGeatkSdDs7fXV5fy_927Dsgakp4A,24567
flatbuffers/compat.py,sha256=ihBSpWDCSL-vgLSyZtcu8LX3ZI3wz9LhtqItY2GQZgg,2373
flatbuffers/encode.py,sha256=2Or3mgWRAkJiWg-GgYasDU4zIHpQU3W06fmIhwbz5uM,1550
flatbuffers/flexbuffers.py,sha256=yF8Wr4Lo8WJb-pj9NNaIYxLwzlHHyTroM0iO8fyDwbU,44454
flatbuffers/number_types.py,sha256=ijO0QcJiuxlQegoBOed0v9m0DdzTZHWxpTBZUqzsWHA,3762
flatbuffers/packer.py,sha256=LNWym8YgFRqHjcPeGpYY3inCGWH6XnbkQKtAPtFEVas,1164
flatbuffers/table.py,sha256=ciYTmq_CzAuYpb3KAVnl75M84ieChfbyKne-dFHzwwU,4818
flatbuffers/util.py,sha256=mRVQ1VoHp0MJMNtRTUGVzALwN4T_C-U14tMbj99py2A,1608

View File

@@ -0,0 +1,6 @@
Wheel-Version: 1.0
Generator: setuptools (80.9.0)
Root-Is-Purelib: true
Tag: py2-none-any
Tag: py3-none-any

View File

@@ -0,0 +1,19 @@
# Copyright 2014 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from . import util
from ._version import __version__
from .builder import Builder
from .compat import range_func as compat_range
from .table import Table

View File

@@ -0,0 +1,17 @@
# Copyright 2019 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Placeholder, to be updated during the release process
# by the setup.py
__version__ = "25.12.19"

View File

@@ -0,0 +1,858 @@
# Copyright 2014 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import warnings
from . import compat
from . import encode
from . import number_types as N
from . import packer
from .compat import memoryview_type
from .compat import NumpyRequiredForThisFeature, import_numpy
from .compat import range_func
from .number_types import (SOffsetTFlags, UOffsetTFlags, VOffsetTFlags)
np = import_numpy()
## @file
## @addtogroup flatbuffers_python_api
## @{
## @cond FLATBUFFERS_INTERNAL
class OffsetArithmeticError(RuntimeError):
"""Error caused by an Offset arithmetic error.
Probably caused by bad writing of fields. This is considered an unreachable
situation in normal circumstances.
"""
pass
class IsNotNestedError(RuntimeError):
"""Error caused by using a Builder to write Object data when not inside
an Object.
"""
pass
class IsNestedError(RuntimeError):
"""Error caused by using a Builder to begin an Object when an Object is
already being built.
"""
pass
class StructIsNotInlineError(RuntimeError):
"""Error caused by using a Builder to write a Struct at a location that
is not the current Offset.
"""
pass
class BuilderSizeError(RuntimeError):
"""Error caused by causing a Builder to exceed the hardcoded limit of 2
gigabytes.
"""
pass
class BuilderNotFinishedError(RuntimeError):
"""Error caused by not calling `Finish` before calling `Output`."""
pass
class EndVectorLengthMismatched(RuntimeError):
"""The number of elements passed to EndVector does not match the number
specified in StartVector.
"""
pass
# VtableMetadataFields is the count of metadata fields in each vtable.
VtableMetadataFields = 2
## @endcond
class Builder(object):
"""A Builder is used to construct one or more FlatBuffers.
Typically, Builder objects will be used from code generated by the `flatc`
compiler.
A Builder constructs byte buffers in a last-first manner for simplicity and
performance during reading.
Internally, a Builder is a state machine for creating FlatBuffer objects.
It holds the following internal state:
- Bytes: an array of bytes.
- current_vtable: a list of integers.
- vtables: a hash of vtable entries.
Attributes:
Bytes: The internal `bytearray` for the Builder.
finished: A boolean determining if the Builder has been finalized.
"""
## @cond FLATBUFFERS_INTENRAL
__slots__ = (
"Bytes",
"current_vtable",
"head",
"minalign",
"objectEnd",
"vtables",
"nested",
"forceDefaults",
"finished",
"vectorNumElems",
"sharedStrings",
)
"""Maximum buffer size constant, in bytes.
Builder will never allow it's buffer grow over this size.
Currently equals 2Gb.
"""
MAX_BUFFER_SIZE = 2**31
## @endcond
def __init__(self, initialSize=1024):
"""Initializes a Builder of size `initial_size`.
The internal buffer is grown as needed.
"""
if not (0 <= initialSize <= Builder.MAX_BUFFER_SIZE):
msg = "flatbuffers: Cannot create Builder larger than 2 gigabytes."
raise BuilderSizeError(msg)
self.Bytes = bytearray(initialSize)
## @cond FLATBUFFERS_INTERNAL
self.current_vtable = None
self.head = UOffsetTFlags.py_type(initialSize)
self.minalign = 1
self.objectEnd = None
self.vtables = {}
self.nested = False
self.forceDefaults = False
self.sharedStrings = None
## @endcond
self.finished = False
def Clear(self):
## @cond FLATBUFFERS_INTERNAL
self.current_vtable = None
self.head = len(self.Bytes)
self.minalign = 1
self.objectEnd = None
self.vtables = {}
self.nested = False
self.forceDefaults = False
self.sharedStrings = None
self.vectorNumElems = None
## @endcond
self.finished = False
def Output(self):
"""Return the portion of the buffer that has been used for writing data.
This is the typical way to access the FlatBuffer data inside the
builder. If you try to access `Builder.Bytes` directly, you would need
to manually index it with `Head()`, since the buffer is constructed
backwards.
It raises BuilderNotFinishedError if the buffer has not been finished
with `Finish`.
"""
if not self.finished:
raise BuilderNotFinishedError()
return self.Bytes[self.head :]
## @cond FLATBUFFERS_INTERNAL
def StartObject(self, numfields):
"""StartObject initializes bookkeeping for writing a new object."""
self.assertNotNested()
# use 32-bit offsets so that arithmetic doesn't overflow.
self.current_vtable = [0] * numfields
self.objectEnd = self.Offset()
self.nested = True
def WriteVtable(self):
"""WriteVtable serializes the vtable for the current object, if needed.
Before writing out the vtable, this checks pre-existing vtables for
equality to this one. If an equal vtable is found, point the object to
the existing vtable and return.
Because vtable values are sensitive to alignment of object data, not
all logically-equal vtables will be deduplicated.
A vtable has the following format:
<VOffsetT: size of the vtable in bytes, including this value>
<VOffsetT: size of the object in bytes, including the vtable offset>
<VOffsetT: offset for a field> * N, where N is the number of fields
in the schema for this type. Includes deprecated fields.
Thus, a vtable is made of 2 + N elements, each VOffsetT bytes wide.
An object has the following format:
<SOffsetT: offset to this object's vtable (may be negative)>
<byte: data>+
"""
# Prepend a zero scalar to the object. Later in this function we'll
# write an offset here that points to the object's vtable:
self.PrependSOffsetTRelative(0)
objectOffset = self.Offset()
vtKey = []
trim = True
for elem in reversed(self.current_vtable):
if elem == 0:
if trim:
continue
else:
elem = objectOffset - elem
trim = False
vtKey.append(elem)
objectSize = UOffsetTFlags.py_type(objectOffset - self.objectEnd)
vtKey.append(objectSize)
vtKey = tuple(vtKey)
# calculate the size of the object
vt2Offset = self.vtables.get(vtKey)
if vt2Offset is None:
# Did not find a vtable, so write this one to the buffer.
# Write out the current vtable in reverse , because
# serialization occurs in last-first order:
i = len(self.current_vtable) - 1
trailing = 0
trim = True
while i >= 0:
off = 0
elem = self.current_vtable[i]
i -= 1
if elem == 0:
if trim:
trailing += 1
continue
else:
# Forward reference to field;
# use 32bit number to ensure no overflow:
off = objectOffset - elem
trim = False
self.PrependVOffsetT(off)
# The two metadata fields are written last.
# First, store the object bytesize:
self.PrependVOffsetT(VOffsetTFlags.py_type(objectSize))
# Second, store the vtable bytesize:
vBytes = len(self.current_vtable) - trailing + VtableMetadataFields
vBytes *= N.VOffsetTFlags.bytewidth
self.PrependVOffsetT(VOffsetTFlags.py_type(vBytes))
# Next, write the offset to the new vtable in the
# already-allocated SOffsetT at the beginning of this object:
objectStart = SOffsetTFlags.py_type(len(self.Bytes) - objectOffset)
encode.Write(
packer.soffset,
self.Bytes,
objectStart,
SOffsetTFlags.py_type(self.Offset() - objectOffset),
)
# Finally, store this vtable in memory for future
# deduplication:
self.vtables[vtKey] = self.Offset()
else:
# Found a duplicate vtable.
objectStart = SOffsetTFlags.py_type(len(self.Bytes) - objectOffset)
self.head = UOffsetTFlags.py_type(objectStart)
# Write the offset to the found vtable in the
# already-allocated SOffsetT at the beginning of this object:
encode.Write(
packer.soffset,
self.Bytes,
self.Head(),
SOffsetTFlags.py_type(vt2Offset - objectOffset),
)
self.current_vtable = None
return objectOffset
def EndObject(self):
"""EndObject writes data necessary to finish object construction."""
self.assertNested()
self.nested = False
return self.WriteVtable()
def GrowByteBuffer(self):
"""Doubles the size of the byteslice, and copies the old data towards
the end of the new buffer (since we build the buffer backwards).
"""
if len(self.Bytes) == Builder.MAX_BUFFER_SIZE:
msg = "flatbuffers: cannot grow buffer beyond 2 gigabytes"
raise BuilderSizeError(msg)
newSize = min(len(self.Bytes) * 2, Builder.MAX_BUFFER_SIZE)
if newSize == 0:
newSize = 1
bytes2 = bytearray(newSize)
bytes2[newSize - len(self.Bytes) :] = self.Bytes
self.Bytes = bytes2
## @endcond
def Head(self):
"""Get the start of useful data in the underlying byte buffer.
Note: unlike other functions, this value is interpreted as from the
left.
"""
## @cond FLATBUFFERS_INTERNAL
return self.head
## @endcond
## @cond FLATBUFFERS_INTERNAL
def Offset(self):
"""Offset relative to the end of the buffer."""
return len(self.Bytes) - self.head
def Pad(self, n):
"""Pad places zeros at the current offset."""
if n <= 0:
return
new_head = self.head - n
self.Bytes[new_head : self.head] = b"\x00" * n
self.head = new_head
def Prep(self, size, additionalBytes):
"""Prep prepares to write an element of `size` after `additional_bytes`
have been written, e.g. if you write a string, you need to align
such the int length field is aligned to SizeInt32, and the string
data follows it directly.
If all you need to do is align, `additionalBytes` will be 0.
"""
# Track the biggest thing we've ever aligned to.
if size > self.minalign:
self.minalign = size
# Find the amount of alignment needed such that `size` is properly
# aligned after `additionalBytes`:
head = self.head
buf_len = len(self.Bytes)
alignSize = (~(buf_len - head + additionalBytes)) + 1
alignSize &= size - 1
# Reallocate the buffer if needed:
needed = alignSize + size + additionalBytes
while head < needed:
oldBufSize = buf_len
self.GrowByteBuffer()
buf_len = len(self.Bytes)
head += buf_len - oldBufSize
self.head = head
self.Pad(alignSize)
def PrependSOffsetTRelative(self, off):
"""PrependSOffsetTRelative prepends an SOffsetT, relative to where it
will be written.
"""
# Ensure alignment is already done:
self.Prep(N.SOffsetTFlags.bytewidth, 0)
if not (off <= self.Offset()):
msg = "flatbuffers: Offset arithmetic error."
raise OffsetArithmeticError(msg)
off2 = self.Offset() - off + N.SOffsetTFlags.bytewidth
self.PlaceSOffsetT(off2)
## @endcond
def PrependUOffsetTRelative(self, off):
"""Prepends an unsigned offset into vector data, relative to where it
will be written.
"""
# Ensure alignment is already done:
self.Prep(N.UOffsetTFlags.bytewidth, 0)
if not (off <= self.Offset()):
msg = "flatbuffers: Offset arithmetic error."
raise OffsetArithmeticError(msg)
off2 = self.Offset() - off + N.UOffsetTFlags.bytewidth
self.PlaceUOffsetT(off2)
## @cond FLATBUFFERS_INTERNAL
def StartVector(self, elemSize, numElems, alignment):
"""StartVector initializes bookkeeping for writing a new vector.
A vector has the following format:
- <UOffsetT: number of elements in this vector>
- <T: data>+, where T is the type of elements of this vector.
"""
self.assertNotNested()
self.nested = True
self.vectorNumElems = numElems
self.Prep(N.Uint32Flags.bytewidth, elemSize * numElems)
self.Prep(alignment, elemSize * numElems) # In case alignment > int.
return self.Offset()
## @endcond
def EndVector(self, numElems=None):
"""EndVector writes data necessary to finish vector construction."""
self.assertNested()
## @cond FLATBUFFERS_INTERNAL
self.nested = False
## @endcond
if numElems:
warnings.warn("numElems is deprecated.", DeprecationWarning, stacklevel=2)
if numElems != self.vectorNumElems:
raise EndVectorLengthMismatched()
# we already made space for this, so write without PrependUint32
self.PlaceUOffsetT(self.vectorNumElems)
self.vectorNumElems = None
return self.Offset()
def CreateSharedString(self, s, encoding="utf-8", errors="strict"):
"""CreateSharedString checks if the string is already written to the buffer
before calling CreateString.
"""
if not self.sharedStrings:
self.sharedStrings = {}
elif s in self.sharedStrings:
return self.sharedStrings[s]
off = self.CreateString(s, encoding, errors)
self.sharedStrings[s] = off
return off
def CreateString(self, s, encoding="utf-8", errors="strict"):
"""CreateString writes a null-terminated byte string as a vector."""
self.assertNotNested()
## @cond FLATBUFFERS_INTERNAL
self.nested = True
## @endcond
if isinstance(s, compat.string_types):
x = s.encode(encoding, errors)
elif isinstance(s, compat.binary_types):
x = s
else:
raise TypeError("non-string passed to CreateString")
payload_len = len(x)
self.Prep(N.UOffsetTFlags.bytewidth, (payload_len + 1) * N.Uint8Flags.bytewidth)
self.Place(0, N.Uint8Flags)
new_head = self.head - payload_len
self.head = new_head
self.Bytes[new_head : new_head + payload_len] = x
self.vectorNumElems = payload_len
return self.EndVector()
def CreateByteVector(self, x):
"""CreateString writes a byte vector."""
self.assertNotNested()
## @cond FLATBUFFERS_INTERNAL
self.nested = True
## @endcond
if not isinstance(x, compat.binary_types):
raise TypeError("non-byte vector passed to CreateByteVector")
data_len = len(x)
self.Prep(N.UOffsetTFlags.bytewidth, data_len * N.Uint8Flags.bytewidth)
new_head = self.head - data_len
self.head = new_head
self.Bytes[new_head : new_head + data_len] = x
self.vectorNumElems = data_len
return self.EndVector()
def CreateNumpyVector(self, x):
"""CreateNumpyVector writes a numpy array into the buffer."""
if np is None:
# Numpy is required for this feature
raise NumpyRequiredForThisFeature("Numpy was not found.")
if not isinstance(x, np.ndarray):
raise TypeError("non-numpy-ndarray passed to CreateNumpyVector")
if x.dtype.kind not in ["b", "i", "u", "f"]:
raise TypeError("numpy-ndarray holds elements of unsupported datatype")
if x.ndim > 1:
raise TypeError("multidimensional-ndarray passed to CreateNumpyVector")
self.StartVector(x.itemsize, x.size, x.dtype.alignment)
# Ensure little endian byte ordering
if x.dtype.str[0] == "<":
x_lend = x
else:
x_lend = x.byteswap(inplace=False)
# tobytes ensures c_contiguous ordering
payload = x_lend.tobytes(order="C")
# Calculate total length
payload_len = len(payload)
new_head = self.head - payload_len
self.head = new_head
self.Bytes[new_head : new_head + payload_len] = payload
self.vectorNumElems = x.size
return self.EndVector()
## @cond FLATBUFFERS_INTERNAL
def assertNested(self):
"""Check that we are in the process of building an object."""
if not self.nested:
raise IsNotNestedError()
def assertNotNested(self):
"""Check that no other objects are being built while making this object.
If not, raise an exception.
"""
if self.nested:
raise IsNestedError()
def assertStructIsInline(self, obj):
"""Structs are always stored inline, so need to be created right
where they are used. You'll get this error if you created it
elsewhere.
"""
N.enforce_number(obj, N.UOffsetTFlags)
if obj != self.Offset():
msg = (
"flatbuffers: Tried to write a Struct at an Offset that "
"is different from the current Offset of the Builder."
)
raise StructIsNotInlineError(msg)
def Slot(self, slotnum):
"""Slot sets the vtable key `voffset` to the current location in the
buffer.
"""
self.assertNested()
self.current_vtable[slotnum] = self.Offset()
## @endcond
def __Finish(self, rootTable, sizePrefix, file_identifier=None):
"""Finish finalizes a buffer, pointing to the given `rootTable`."""
N.enforce_number(rootTable, N.UOffsetTFlags)
prepSize = N.UOffsetTFlags.bytewidth
if file_identifier is not None:
prepSize += N.Int32Flags.bytewidth
if sizePrefix:
prepSize += N.Int32Flags.bytewidth
self.Prep(self.minalign, prepSize)
if file_identifier is not None:
self.Prep(N.UOffsetTFlags.bytewidth, encode.FILE_IDENTIFIER_LENGTH)
# Convert bytes object file_identifier to an array of 4 8-bit integers,
# and use big-endian to enforce size compliance.
# https://docs.python.org/2/library/struct.html#format-characters
file_identifier = N.struct.unpack(">BBBB", file_identifier)
for i in range(encode.FILE_IDENTIFIER_LENGTH - 1, -1, -1):
# Place the bytes of the file_identifer in reverse order:
self.Place(file_identifier[i], N.Uint8Flags)
self.PrependUOffsetTRelative(rootTable)
if sizePrefix:
size = len(self.Bytes) - self.head
N.enforce_number(size, N.Int32Flags)
self.PrependInt32(size)
self.finished = True
return self.head
def Finish(self, rootTable, file_identifier=None):
"""Finish finalizes a buffer, pointing to the given `rootTable`."""
return self.__Finish(rootTable, False, file_identifier=file_identifier)
def FinishSizePrefixed(self, rootTable, file_identifier=None):
"""Finish finalizes a buffer, pointing to the given `rootTable`,
with the size prefixed.
"""
return self.__Finish(rootTable, True, file_identifier=file_identifier)
## @cond FLATBUFFERS_INTERNAL
def Prepend(self, flags, off):
self.Prep(flags.bytewidth, 0)
self.Place(off, flags)
def PrependSlot(self, flags, o, x, d):
if x is not None:
N.enforce_number(x, flags)
if d is not None:
N.enforce_number(d, flags)
if x != d or (self.forceDefaults and d is not None):
self.Prepend(flags, x)
self.Slot(o)
def PrependBoolSlot(self, *args):
self.PrependSlot(N.BoolFlags, *args)
def PrependByteSlot(self, *args):
self.PrependSlot(N.Uint8Flags, *args)
def PrependUint8Slot(self, *args):
self.PrependSlot(N.Uint8Flags, *args)
def PrependUint16Slot(self, *args):
self.PrependSlot(N.Uint16Flags, *args)
def PrependUint32Slot(self, *args):
self.PrependSlot(N.Uint32Flags, *args)
def PrependUint64Slot(self, *args):
self.PrependSlot(N.Uint64Flags, *args)
def PrependInt8Slot(self, *args):
self.PrependSlot(N.Int8Flags, *args)
def PrependInt16Slot(self, *args):
self.PrependSlot(N.Int16Flags, *args)
def PrependInt32Slot(self, *args):
self.PrependSlot(N.Int32Flags, *args)
def PrependInt64Slot(self, *args):
self.PrependSlot(N.Int64Flags, *args)
def PrependFloat32Slot(self, *args):
self.PrependSlot(N.Float32Flags, *args)
def PrependFloat64Slot(self, *args):
self.PrependSlot(N.Float64Flags, *args)
def PrependUOffsetTRelativeSlot(self, o, x, d):
"""PrependUOffsetTRelativeSlot prepends an UOffsetT onto the object at
vtable slot `o`. If value `x` equals default `d`, then the slot will
be set to zero and no other data will be written.
"""
if x != d or self.forceDefaults:
self.PrependUOffsetTRelative(x)
self.Slot(o)
def PrependStructSlot(self, v, x, d):
"""PrependStructSlot prepends a struct onto the object at vtable slot `o`.
Structs are stored inline, so nothing additional is being added. In
generated code, `d` is always 0.
"""
N.enforce_number(d, N.UOffsetTFlags)
if x != d:
self.assertStructIsInline(x)
self.Slot(v)
## @endcond
def PrependBool(self, x):
"""Prepend a `bool` to the Builder buffer.
Note: aligns and checks for space.
"""
self.Prepend(N.BoolFlags, x)
def PrependByte(self, x):
"""Prepend a `byte` to the Builder buffer.
Note: aligns and checks for space.
"""
self.Prepend(N.Uint8Flags, x)
def PrependUint8(self, x):
"""Prepend an `uint8` to the Builder buffer.
Note: aligns and checks for space.
"""
self.Prepend(N.Uint8Flags, x)
def PrependUint16(self, x):
"""Prepend an `uint16` to the Builder buffer.
Note: aligns and checks for space.
"""
self.Prepend(N.Uint16Flags, x)
def PrependUint32(self, x):
"""Prepend an `uint32` to the Builder buffer.
Note: aligns and checks for space.
"""
self.Prepend(N.Uint32Flags, x)
def PrependUint64(self, x):
"""Prepend an `uint64` to the Builder buffer.
Note: aligns and checks for space.
"""
self.Prepend(N.Uint64Flags, x)
def PrependInt8(self, x):
"""Prepend an `int8` to the Builder buffer.
Note: aligns and checks for space.
"""
self.Prepend(N.Int8Flags, x)
def PrependInt16(self, x):
"""Prepend an `int16` to the Builder buffer.
Note: aligns and checks for space.
"""
self.Prepend(N.Int16Flags, x)
def PrependInt32(self, x):
"""Prepend an `int32` to the Builder buffer.
Note: aligns and checks for space.
"""
self.Prepend(N.Int32Flags, x)
def PrependInt64(self, x):
"""Prepend an `int64` to the Builder buffer.
Note: aligns and checks for space.
"""
self.Prepend(N.Int64Flags, x)
def PrependFloat32(self, x):
"""Prepend a `float32` to the Builder buffer.
Note: aligns and checks for space.
"""
self.Prepend(N.Float32Flags, x)
def PrependFloat64(self, x):
"""Prepend a `float64` to the Builder buffer.
Note: aligns and checks for space.
"""
self.Prepend(N.Float64Flags, x)
def ForceDefaults(self, forceDefaults):
"""In order to save space, fields that are set to their default value
don't get serialized into the buffer. Forcing defaults provides a
way to manually disable this optimization. When set to `True`, will
always serialize default values.
"""
self.forceDefaults = forceDefaults
##############################################################
## @cond FLATBUFFERS_INTERNAL
def PrependVOffsetT(self, x):
self.Prepend(N.VOffsetTFlags, x)
def Place(self, x, flags):
"""Place prepends a value specified by `flags` to the Builder,
without checking for available space.
"""
N.enforce_number(x, flags)
new_head = self.head - flags.bytewidth
self.head = new_head
encode.Write(flags.packer_type, self.Bytes, new_head, x)
def PlaceVOffsetT(self, x):
"""PlaceVOffsetT prepends a VOffsetT to the Builder, without checking
for space.
"""
N.enforce_number(x, N.VOffsetTFlags)
new_head = self.head - N.VOffsetTFlags.bytewidth
self.head = new_head
encode.Write(packer.voffset, self.Bytes, new_head, x)
def PlaceSOffsetT(self, x):
"""PlaceSOffsetT prepends a SOffsetT to the Builder, without checking
for space.
"""
N.enforce_number(x, N.SOffsetTFlags)
new_head = self.head - N.SOffsetTFlags.bytewidth
self.head = new_head
encode.Write(packer.soffset, self.Bytes, new_head, x)
def PlaceUOffsetT(self, x):
"""PlaceUOffsetT prepends a UOffsetT to the Builder, without checking
for space.
"""
N.enforce_number(x, N.UOffsetTFlags)
new_head = self.head - N.UOffsetTFlags.bytewidth
self.head = new_head
encode.Write(packer.uoffset, self.Bytes, new_head, x)
## @endcond
## @}

View File

@@ -0,0 +1,91 @@
# Copyright 2016 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""A tiny version of `six` to help with backwards compability.
Also includes compatibility helpers for numpy.
"""
import sys
PY2 = sys.version_info[0] == 2
PY26 = sys.version_info[0:2] == (2, 6)
PY27 = sys.version_info[0:2] == (2, 7)
PY275 = sys.version_info[0:3] >= (2, 7, 5)
PY3 = sys.version_info[0] == 3
PY34 = sys.version_info[0:2] >= (3, 4)
if PY3:
import importlib.machinery
string_types = (str,)
binary_types = (bytes, bytearray)
range_func = range
memoryview_type = memoryview
struct_bool_decl = "?"
else:
import imp
string_types = (unicode,)
if PY26 or PY27:
binary_types = (str, bytearray)
else:
binary_types = (str,)
range_func = xrange
if PY26 or (PY27 and not PY275):
memoryview_type = buffer
struct_bool_decl = "<b"
else:
memoryview_type = memoryview
struct_bool_decl = "?"
# Helper functions to facilitate making numpy optional instead of required
def import_numpy():
"""Returns the numpy module if it exists on the system,
otherwise returns None.
"""
if PY3:
numpy_exists = importlib.machinery.PathFinder.find_spec("numpy") is not None
else:
try:
imp.find_module("numpy")
numpy_exists = True
except ImportError:
numpy_exists = False
if numpy_exists:
# We do this outside of try/except block in case numpy exists
# but is not installed correctly. We do not want to catch an
# incorrect installation which would manifest as an
# ImportError.
import numpy as np
else:
np = None
return np
class NumpyRequiredForThisFeature(RuntimeError):
"""Error raised when user tries to use a feature that
requires numpy without having numpy installed.
"""
pass
# NOTE: Future Jython support may require code here (look at `six`).

View File

@@ -0,0 +1,45 @@
# Copyright 2014 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from . import number_types as N
from . import packer
from .compat import memoryview_type
from .compat import NumpyRequiredForThisFeature, import_numpy
np = import_numpy()
FILE_IDENTIFIER_LENGTH = 4
def Get(packer_type, buf, head):
"""Get decodes a value at buf[head] using `packer_type`."""
return packer_type.unpack_from(memoryview_type(buf), head)[0]
def GetVectorAsNumpy(numpy_type, buf, count, offset):
"""GetVecAsNumpy decodes values starting at buf[head] as
`numpy_type`, where `numpy_type` is a numpy dtype.
"""
if np is not None:
# TODO: could set .flags.writeable = False to make users jump through
# hoops before modifying...
return np.frombuffer(buf, dtype=numpy_type, count=count, offset=offset)
else:
raise NumpyRequiredForThisFeature('Numpy was not found.')
def Write(packer_type, buf, head, n):
"""Write encodes `n` at buf[head] using `packer_type`."""
packer_type.pack_into(buf, head, n)

View File

@@ -0,0 +1,182 @@
# Copyright 2014 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import collections
import struct
from . import packer
from .compat import NumpyRequiredForThisFeature, import_numpy
np = import_numpy()
# For reference, see:
# https://docs.python.org/2/library/ctypes.html#ctypes-fundamental-data-types-2
# These classes could be collections.namedtuple instances, but those are new
# in 2.6 and we want to work towards 2.5 compatability.
class BoolFlags(object):
bytewidth = 1
min_val = False
max_val = True
py_type = bool
name = "bool"
packer_type = packer.boolean
class Uint8Flags(object):
bytewidth = 1
min_val = 0
max_val = (2**8) - 1
py_type = int
name = "uint8"
packer_type = packer.uint8
class Uint16Flags(object):
bytewidth = 2
min_val = 0
max_val = (2**16) - 1
py_type = int
name = "uint16"
packer_type = packer.uint16
class Uint32Flags(object):
bytewidth = 4
min_val = 0
max_val = (2**32) - 1
py_type = int
name = "uint32"
packer_type = packer.uint32
class Uint64Flags(object):
bytewidth = 8
min_val = 0
max_val = (2**64) - 1
py_type = int
name = "uint64"
packer_type = packer.uint64
class Int8Flags(object):
bytewidth = 1
min_val = -(2**7)
max_val = (2**7) - 1
py_type = int
name = "int8"
packer_type = packer.int8
class Int16Flags(object):
bytewidth = 2
min_val = -(2**15)
max_val = (2**15) - 1
py_type = int
name = "int16"
packer_type = packer.int16
class Int32Flags(object):
bytewidth = 4
min_val = -(2**31)
max_val = (2**31) - 1
py_type = int
name = "int32"
packer_type = packer.int32
class Int64Flags(object):
bytewidth = 8
min_val = -(2**63)
max_val = (2**63) - 1
py_type = int
name = "int64"
packer_type = packer.int64
class Float32Flags(object):
bytewidth = 4
min_val = None
max_val = None
py_type = float
name = "float32"
packer_type = packer.float32
class Float64Flags(object):
bytewidth = 8
min_val = None
max_val = None
py_type = float
name = "float64"
packer_type = packer.float64
class SOffsetTFlags(Int32Flags):
pass
class UOffsetTFlags(Uint32Flags):
pass
class VOffsetTFlags(Uint16Flags):
pass
def valid_number(n, flags):
if flags.min_val is None and flags.max_val is None:
return True
return flags.min_val <= n <= flags.max_val
def enforce_number(n, flags):
if flags.min_val is None and flags.max_val is None:
return
if not flags.min_val <= n <= flags.max_val:
raise TypeError("bad number %s for type %s" % (str(n), flags.name))
def float32_to_uint32(n):
packed = struct.pack("<1f", n)
(converted,) = struct.unpack("<1L", packed)
return converted
def uint32_to_float32(n):
packed = struct.pack("<1L", n)
(unpacked,) = struct.unpack("<1f", packed)
return unpacked
def float64_to_uint64(n):
packed = struct.pack("<1d", n)
(converted,) = struct.unpack("<1Q", packed)
return converted
def uint64_to_float64(n):
packed = struct.pack("<1Q", n)
(unpacked,) = struct.unpack("<1d", packed)
return unpacked
def to_numpy_type(number_type):
if np is not None:
return np.dtype(number_type.name).newbyteorder("<")
else:
raise NumpyRequiredForThisFeature("Numpy was not found.")

View File

@@ -0,0 +1,41 @@
# Copyright 2016 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Provide pre-compiled struct packers for encoding and decoding.
See: https://docs.python.org/2/library/struct.html#format-characters
"""
import struct
from . import compat
boolean = struct.Struct(compat.struct_bool_decl)
uint8 = struct.Struct("<B")
uint16 = struct.Struct("<H")
uint32 = struct.Struct("<I")
uint64 = struct.Struct("<Q")
int8 = struct.Struct("<b")
int16 = struct.Struct("<h")
int32 = struct.Struct("<i")
int64 = struct.Struct("<q")
float32 = struct.Struct("<f")
float64 = struct.Struct("<d")
uoffset = uint32
soffset = int32
voffset = uint16

View File

@@ -0,0 +1,148 @@
# Copyright 2014 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from . import encode
from . import number_types as N
class Table(object):
"""Table wraps a byte slice and provides read access to its data.
The variable `Pos` indicates the root of the FlatBuffers object therein.
"""
__slots__ = ("Bytes", "Pos")
def __init__(self, buf, pos):
N.enforce_number(pos, N.UOffsetTFlags)
self.Bytes = buf
self.Pos = pos
def Offset(self, vtableOffset):
"""Offset provides access into the Table's vtable.
Deprecated fields are ignored by checking the vtable's length.
"""
vtable = self.Pos - self.Get(N.SOffsetTFlags, self.Pos)
vtableEnd = self.Get(N.VOffsetTFlags, vtable)
if vtableOffset < vtableEnd:
return self.Get(N.VOffsetTFlags, vtable + vtableOffset)
return 0
def Indirect(self, off):
"""Indirect retrieves the relative offset stored at `offset`."""
N.enforce_number(off, N.UOffsetTFlags)
return off + encode.Get(N.UOffsetTFlags.packer_type, self.Bytes, off)
def String(self, off):
"""String gets a string from data stored inside the flatbuffer."""
N.enforce_number(off, N.UOffsetTFlags)
off += encode.Get(N.UOffsetTFlags.packer_type, self.Bytes, off)
start = off + N.UOffsetTFlags.bytewidth
length = encode.Get(N.UOffsetTFlags.packer_type, self.Bytes, off)
return bytes(self.Bytes[start : start + length])
def VectorLen(self, off):
"""VectorLen retrieves the length of the vector whose offset is stored
at "off" in this object.
"""
N.enforce_number(off, N.UOffsetTFlags)
off += self.Pos
off += encode.Get(N.UOffsetTFlags.packer_type, self.Bytes, off)
ret = encode.Get(N.UOffsetTFlags.packer_type, self.Bytes, off)
return ret
def Vector(self, off):
"""Vector retrieves the start of data of the vector whose offset is
stored at "off" in this object.
"""
N.enforce_number(off, N.UOffsetTFlags)
off += self.Pos
x = off + self.Get(N.UOffsetTFlags, off)
# data starts after metadata containing the vector length
x += N.UOffsetTFlags.bytewidth
return x
def Union(self, t2, off):
"""Union initializes any Table-derived type to point to the union at
the given offset.
"""
assert type(t2) is Table
N.enforce_number(off, N.UOffsetTFlags)
off += self.Pos
t2.Pos = off + self.Get(N.UOffsetTFlags, off)
t2.Bytes = self.Bytes
def Get(self, flags, off):
"""Get retrieves a value of the type specified by `flags` at the
given offset.
"""
N.enforce_number(off, N.UOffsetTFlags)
return flags.py_type(encode.Get(flags.packer_type, self.Bytes, off))
def GetSlot(self, slot, d, validator_flags):
N.enforce_number(slot, N.VOffsetTFlags)
if validator_flags is not None:
N.enforce_number(d, validator_flags)
off = self.Offset(slot)
if off == 0:
return d
return self.Get(validator_flags, self.Pos + off)
def GetVectorAsNumpy(self, flags, off):
"""GetVectorAsNumpy returns the vector that starts at `Vector(off)`
as a numpy array with the type specified by `flags`. The array is
a `view` into Bytes, so modifying the returned array will
modify Bytes in place.
"""
offset = self.Vector(off)
length = self.VectorLen(off) # TODO: length accounts for bytewidth, right?
numpy_dtype = N.to_numpy_type(flags)
return encode.GetVectorAsNumpy(numpy_dtype, self.Bytes, length, offset)
def GetArrayAsNumpy(self, flags, off, length):
"""GetArrayAsNumpy returns the array with fixed width that starts at `Vector(offset)`
with length `length` as a numpy array with the type specified by `flags`.
The
array is a `view` into Bytes so modifying the returned will modify Bytes in
place.
"""
numpy_dtype = N.to_numpy_type(flags)
return encode.GetVectorAsNumpy(numpy_dtype, self.Bytes, length, off)
def GetVOffsetTSlot(self, slot, d):
"""GetVOffsetTSlot retrieves the VOffsetT that the given vtable location
points to. If the vtable value is zero, the default value `d`
will be returned.
"""
N.enforce_number(slot, N.VOffsetTFlags)
N.enforce_number(d, N.VOffsetTFlags)
off = self.Offset(slot)
if off == 0:
return d
return off

View File

@@ -0,0 +1,47 @@
# Copyright 2017 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from . import encode
from . import number_types
from . import packer
def GetSizePrefix(buf, offset):
"""Extract the size prefix from a buffer."""
return encode.Get(packer.int32, buf, offset)
def GetBufferIdentifier(buf, offset, size_prefixed=False):
"""Extract the file_identifier from a buffer"""
if size_prefixed:
# increase offset by size of UOffsetTFlags
offset += number_types.UOffsetTFlags.bytewidth
# increase offset by size of root table pointer
offset += number_types.UOffsetTFlags.bytewidth
# end of FILE_IDENTIFIER
end = offset + encode.FILE_IDENTIFIER_LENGTH
return buf[offset:end]
def BufferHasIdentifier(buf, offset, file_identifier, size_prefixed=False):
got = GetBufferIdentifier(buf, offset, size_prefixed=size_prefixed)
return got == file_identifier
def RemoveSizePrefix(buf, offset):
"""Create a slice of a size-prefixed buffer that has
its position advanced just past the size prefix.
"""
return buf, offset + number_types.Int32Flags.bytewidth

View File

@@ -0,0 +1,10 @@
# Protocol Buffers - Google's data interchange format
# Copyright 2008 Google Inc. All rights reserved.
#
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file or at
# https://developers.google.com/open-source/licenses/bsd
# Copyright 2007 Google Inc. All Rights Reserved.
__version__ = '7.34.0'

View File

@@ -0,0 +1,53 @@
# Protocol Buffers - Google's data interchange format
# Copyright 2008 Google Inc. All rights reserved.
#
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file or at
# https://developers.google.com/open-source/licenses/bsd
"""Contains the Any helper APIs."""
from typing import Optional, TypeVar
from google.protobuf import descriptor
from google.protobuf.message import Message
from google.protobuf.any_pb2 import Any
_MessageT = TypeVar('_MessageT', bound=Message)
def pack(
msg: Message,
type_url_prefix: Optional[str] = 'type.googleapis.com/',
deterministic: Optional[bool] = None,
) -> Any:
any_msg = Any()
any_msg.Pack(
msg=msg, type_url_prefix=type_url_prefix, deterministic=deterministic
)
return any_msg
def unpack(any_msg: Any, msg: Message) -> bool:
return any_msg.Unpack(msg=msg)
def unpack_as(any_msg: Any, message_type: type[_MessageT]) -> _MessageT:
unpacked = message_type()
if unpack(any_msg, unpacked):
return unpacked
else:
raise TypeError(
f'Attempted to unpack {type_name(any_msg)} to'
f' {message_type.__qualname__}'
)
def type_name(any_msg: Any) -> str:
return any_msg.TypeName()
def is_type(any_msg: Any, des: descriptor.Descriptor) -> bool:
return any_msg.Is(des)

View File

@@ -0,0 +1,37 @@
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler. DO NOT EDIT!
# NO CHECKED-IN PROTOBUF GENCODE
# source: google/protobuf/any.proto
# Protobuf Python Version: 7.34.0
"""Generated protocol buffer code."""
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import runtime_version as _runtime_version
from google.protobuf import symbol_database as _symbol_database
from google.protobuf.internal import builder as _builder
_runtime_version.ValidateProtobufRuntimeVersion(
_runtime_version.Domain.PUBLIC,
7,
34,
0,
'',
'google/protobuf/any.proto'
)
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x19google/protobuf/any.proto\x12\x0fgoogle.protobuf\"6\n\x03\x41ny\x12\x19\n\x08type_url\x18\x01 \x01(\tR\x07typeUrl\x12\x14\n\x05value\x18\x02 \x01(\x0cR\x05valueBv\n\x13\x63om.google.protobufB\x08\x41nyProtoP\x01Z,google.golang.org/protobuf/types/known/anypb\xa2\x02\x03GPB\xaa\x02\x1eGoogle.Protobuf.WellKnownTypesb\x06proto3')
_globals = globals()
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'google.protobuf.any_pb2', _globals)
if not _descriptor._USE_C_DESCRIPTORS:
_globals['DESCRIPTOR']._loaded_options = None
_globals['DESCRIPTOR']._serialized_options = b'\n\023com.google.protobufB\010AnyProtoP\001Z,google.golang.org/protobuf/types/known/anypb\242\002\003GPB\252\002\036Google.Protobuf.WellKnownTypes'
_globals['_ANY']._serialized_start=46
_globals['_ANY']._serialized_end=100
# @@protoc_insertion_point(module_scope)

View File

@@ -0,0 +1,47 @@
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler. DO NOT EDIT!
# NO CHECKED-IN PROTOBUF GENCODE
# source: google/protobuf/api.proto
# Protobuf Python Version: 7.34.0
"""Generated protocol buffer code."""
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import runtime_version as _runtime_version
from google.protobuf import symbol_database as _symbol_database
from google.protobuf.internal import builder as _builder
_runtime_version.ValidateProtobufRuntimeVersion(
_runtime_version.Domain.PUBLIC,
7,
34,
0,
'',
'google/protobuf/api.proto'
)
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
from google.protobuf import source_context_pb2 as google_dot_protobuf_dot_source__context__pb2
from google.protobuf import type_pb2 as google_dot_protobuf_dot_type__pb2
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x19google/protobuf/api.proto\x12\x0fgoogle.protobuf\x1a$google/protobuf/source_context.proto\x1a\x1agoogle/protobuf/type.proto\"\xdb\x02\n\x03\x41pi\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x31\n\x07methods\x18\x02 \x03(\x0b\x32\x17.google.protobuf.MethodR\x07methods\x12\x31\n\x07options\x18\x03 \x03(\x0b\x32\x17.google.protobuf.OptionR\x07options\x12\x18\n\x07version\x18\x04 \x01(\tR\x07version\x12\x45\n\x0esource_context\x18\x05 \x01(\x0b\x32\x1e.google.protobuf.SourceContextR\rsourceContext\x12.\n\x06mixins\x18\x06 \x03(\x0b\x32\x16.google.protobuf.MixinR\x06mixins\x12/\n\x06syntax\x18\x07 \x01(\x0e\x32\x17.google.protobuf.SyntaxR\x06syntax\x12\x18\n\x07\x65\x64ition\x18\x08 \x01(\tR\x07\x65\x64ition\"\xd4\x02\n\x06Method\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12(\n\x10request_type_url\x18\x02 \x01(\tR\x0erequestTypeUrl\x12+\n\x11request_streaming\x18\x03 \x01(\x08R\x10requestStreaming\x12*\n\x11response_type_url\x18\x04 \x01(\tR\x0fresponseTypeUrl\x12-\n\x12response_streaming\x18\x05 \x01(\x08R\x11responseStreaming\x12\x31\n\x07options\x18\x06 \x03(\x0b\x32\x17.google.protobuf.OptionR\x07options\x12\x33\n\x06syntax\x18\x07 \x01(\x0e\x32\x17.google.protobuf.SyntaxB\x02\x18\x01R\x06syntax\x12\x1c\n\x07\x65\x64ition\x18\x08 \x01(\tB\x02\x18\x01R\x07\x65\x64ition\"/\n\x05Mixin\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x12\n\x04root\x18\x02 \x01(\tR\x04rootBv\n\x13\x63om.google.protobufB\x08\x41piProtoP\x01Z,google.golang.org/protobuf/types/known/apipb\xa2\x02\x03GPB\xaa\x02\x1eGoogle.Protobuf.WellKnownTypesb\x06proto3')
_globals = globals()
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'google.protobuf.api_pb2', _globals)
if not _descriptor._USE_C_DESCRIPTORS:
_globals['DESCRIPTOR']._loaded_options = None
_globals['DESCRIPTOR']._serialized_options = b'\n\023com.google.protobufB\010ApiProtoP\001Z,google.golang.org/protobuf/types/known/apipb\242\002\003GPB\252\002\036Google.Protobuf.WellKnownTypes'
_globals['_METHOD'].fields_by_name['syntax']._loaded_options = None
_globals['_METHOD'].fields_by_name['syntax']._serialized_options = b'\030\001'
_globals['_METHOD'].fields_by_name['edition']._loaded_options = None
_globals['_METHOD'].fields_by_name['edition']._serialized_options = b'\030\001'
_globals['_API']._serialized_start=113
_globals['_API']._serialized_end=460
_globals['_METHOD']._serialized_start=463
_globals['_METHOD']._serialized_end=803
_globals['_MIXIN']._serialized_start=805
_globals['_MIXIN']._serialized_end=852
# @@protoc_insertion_point(module_scope)

View File

@@ -0,0 +1,46 @@
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler. DO NOT EDIT!
# NO CHECKED-IN PROTOBUF GENCODE
# source: google/protobuf/compiler/plugin.proto
# Protobuf Python Version: 7.34.0
"""Generated protocol buffer code."""
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import runtime_version as _runtime_version
from google.protobuf import symbol_database as _symbol_database
from google.protobuf.internal import builder as _builder
_runtime_version.ValidateProtobufRuntimeVersion(
_runtime_version.Domain.PUBLIC,
7,
34,
0,
'',
'google/protobuf/compiler/plugin.proto'
)
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
from google.protobuf import descriptor_pb2 as google_dot_protobuf_dot_descriptor__pb2
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n%google/protobuf/compiler/plugin.proto\x12\x18google.protobuf.compiler\x1a google/protobuf/descriptor.proto\"c\n\x07Version\x12\x14\n\x05major\x18\x01 \x01(\x05R\x05major\x12\x14\n\x05minor\x18\x02 \x01(\x05R\x05minor\x12\x14\n\x05patch\x18\x03 \x01(\x05R\x05patch\x12\x16\n\x06suffix\x18\x04 \x01(\tR\x06suffix\"\xcf\x02\n\x14\x43odeGeneratorRequest\x12(\n\x10\x66ile_to_generate\x18\x01 \x03(\tR\x0e\x66ileToGenerate\x12\x1c\n\tparameter\x18\x02 \x01(\tR\tparameter\x12\x43\n\nproto_file\x18\x0f \x03(\x0b\x32$.google.protobuf.FileDescriptorProtoR\tprotoFile\x12\\\n\x17source_file_descriptors\x18\x11 \x03(\x0b\x32$.google.protobuf.FileDescriptorProtoR\x15sourceFileDescriptors\x12L\n\x10\x63ompiler_version\x18\x03 \x01(\x0b\x32!.google.protobuf.compiler.VersionR\x0f\x63ompilerVersion\"\x85\x04\n\x15\x43odeGeneratorResponse\x12\x14\n\x05\x65rror\x18\x01 \x01(\tR\x05\x65rror\x12-\n\x12supported_features\x18\x02 \x01(\x04R\x11supportedFeatures\x12\'\n\x0fminimum_edition\x18\x03 \x01(\x05R\x0eminimumEdition\x12\'\n\x0fmaximum_edition\x18\x04 \x01(\x05R\x0emaximumEdition\x12H\n\x04\x66ile\x18\x0f \x03(\x0b\x32\x34.google.protobuf.compiler.CodeGeneratorResponse.FileR\x04\x66ile\x1a\xb1\x01\n\x04\x46ile\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\'\n\x0finsertion_point\x18\x02 \x01(\tR\x0einsertionPoint\x12\x18\n\x07\x63ontent\x18\x0f \x01(\tR\x07\x63ontent\x12R\n\x13generated_code_info\x18\x10 \x01(\x0b\x32\".google.protobuf.GeneratedCodeInfoR\x11generatedCodeInfo\"W\n\x07\x46\x65\x61ture\x12\x10\n\x0c\x46\x45\x41TURE_NONE\x10\x00\x12\x1b\n\x17\x46\x45\x41TURE_PROTO3_OPTIONAL\x10\x01\x12\x1d\n\x19\x46\x45\x41TURE_SUPPORTS_EDITIONS\x10\x02\x42r\n\x1c\x63om.google.protobuf.compilerB\x0cPluginProtosZ)google.golang.org/protobuf/types/pluginpb\xaa\x02\x18Google.Protobuf.Compiler')
_globals = globals()
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'google.protobuf.compiler.plugin_pb2', _globals)
if not _descriptor._USE_C_DESCRIPTORS:
_globals['DESCRIPTOR']._loaded_options = None
_globals['DESCRIPTOR']._serialized_options = b'\n\034com.google.protobuf.compilerB\014PluginProtosZ)google.golang.org/protobuf/types/pluginpb\252\002\030Google.Protobuf.Compiler'
_globals['_VERSION']._serialized_start=101
_globals['_VERSION']._serialized_end=200
_globals['_CODEGENERATORREQUEST']._serialized_start=203
_globals['_CODEGENERATORREQUEST']._serialized_end=538
_globals['_CODEGENERATORRESPONSE']._serialized_start=541
_globals['_CODEGENERATORRESPONSE']._serialized_end=1058
_globals['_CODEGENERATORRESPONSE_FILE']._serialized_start=792
_globals['_CODEGENERATORRESPONSE_FILE']._serialized_end=969
_globals['_CODEGENERATORRESPONSE_FEATURE']._serialized_start=971
_globals['_CODEGENERATORRESPONSE_FEATURE']._serialized_end=1058
# @@protoc_insertion_point(module_scope)

View File

@@ -0,0 +1,172 @@
# Protocol Buffers - Google's data interchange format
# Copyright 2008 Google Inc. All rights reserved.
#
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file or at
# https://developers.google.com/open-source/licenses/bsd
"""Provides a container for DescriptorProtos."""
__author__ = 'matthewtoia@google.com (Matt Toia)'
import warnings
class Error(Exception):
pass
class DescriptorDatabaseConflictingDefinitionError(Error):
"""Raised when a proto is added with the same name & different descriptor."""
class DescriptorDatabase(object):
"""A container accepting FileDescriptorProtos and maps DescriptorProtos."""
def __init__(self):
self._file_desc_protos_by_file = {}
self._file_desc_protos_by_symbol = {}
def Add(self, file_desc_proto):
"""Adds the FileDescriptorProto and its types to this database.
Args:
file_desc_proto: The FileDescriptorProto to add.
Raises:
DescriptorDatabaseConflictingDefinitionError: if an attempt is made to
add a proto with the same name but different definition than an
existing proto in the database.
"""
proto_name = file_desc_proto.name
if proto_name not in self._file_desc_protos_by_file:
self._file_desc_protos_by_file[proto_name] = file_desc_proto
elif self._file_desc_protos_by_file[proto_name] != file_desc_proto:
raise DescriptorDatabaseConflictingDefinitionError(
'%s already added, but with different descriptor.' % proto_name)
else:
return
# Add all the top-level descriptors to the index.
package = file_desc_proto.package
for message in file_desc_proto.message_type:
for name in _ExtractSymbols(message, package):
self._AddSymbol(name, file_desc_proto)
for enum in file_desc_proto.enum_type:
self._AddSymbol(
('.'.join((package, enum.name)) if package else enum.name),
file_desc_proto,
)
for enum_value in enum.value:
self._file_desc_protos_by_symbol[
'.'.join((package, enum_value.name)) if package else enum_value.name
] = file_desc_proto
for extension in file_desc_proto.extension:
self._AddSymbol(
('.'.join((package, extension.name)) if package else extension.name),
file_desc_proto,
)
for service in file_desc_proto.service:
self._AddSymbol(
('.'.join((package, service.name)) if package else service.name),
file_desc_proto,
)
def FindFileByName(self, name):
"""Finds the file descriptor proto by file name.
Typically the file name is a relative path ending to a .proto file. The
proto with the given name will have to have been added to this database
using the Add method or else an error will be raised.
Args:
name: The file name to find.
Returns:
The file descriptor proto matching the name.
Raises:
KeyError if no file by the given name was added.
"""
return self._file_desc_protos_by_file[name]
def FindFileContainingSymbol(self, symbol):
"""Finds the file descriptor proto containing the specified symbol.
The symbol should be a fully qualified name including the file descriptor's
package and any containing messages. Some examples:
'some.package.name.Message'
'some.package.name.Message.NestedEnum'
'some.package.name.Message.some_field'
The file descriptor proto containing the specified symbol must be added to
this database using the Add method or else an error will be raised.
Args:
symbol: The fully qualified symbol name.
Returns:
The file descriptor proto containing the symbol.
Raises:
KeyError if no file contains the specified symbol.
"""
if symbol.count('.') == 1 and symbol[0] == '.':
symbol = symbol.lstrip('.')
warnings.warn(
'Please remove the leading "." when '
'FindFileContainingSymbol, this will turn to error '
'in 2026 Jan.',
RuntimeWarning,
)
try:
return self._file_desc_protos_by_symbol[symbol]
except KeyError:
# Fields, enum values, and nested extensions are not in
# _file_desc_protos_by_symbol. Try to find the top level
# descriptor. Non-existent nested symbol under a valid top level
# descriptor can also be found. The behavior is the same with
# protobuf C++.
top_level, _, _ = symbol.rpartition('.')
try:
return self._file_desc_protos_by_symbol[top_level]
except KeyError:
# Raise the original symbol as a KeyError for better diagnostics.
raise KeyError(symbol)
def FindFileContainingExtension(self, extendee_name, extension_number):
# TODO: implement this API.
return None
def FindAllExtensionNumbers(self, extendee_name):
# TODO: implement this API.
return []
def _AddSymbol(self, name, file_desc_proto):
if name in self._file_desc_protos_by_symbol:
warn_msg = ('Conflict register for file "' + file_desc_proto.name +
'": ' + name +
' is already defined in file "' +
self._file_desc_protos_by_symbol[name].name + '"')
warnings.warn(warn_msg, RuntimeWarning)
self._file_desc_protos_by_symbol[name] = file_desc_proto
def _ExtractSymbols(desc_proto, package):
"""Pulls out all the symbols from a descriptor proto.
Args:
desc_proto: The proto to extract symbols from.
package: The package containing the descriptor type.
Yields:
The fully qualified name found in the descriptor.
"""
message_name = package + '.' + desc_proto.name if package else desc_proto.name
yield message_name
for nested_type in desc_proto.nested_type:
for symbol in _ExtractSymbols(nested_type, message_name):
yield symbol
for enum_type in desc_proto.enum_type:
yield '.'.join((message_name, enum_type.name))

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1,100 @@
# Protocol Buffers - Google's data interchange format
# Copyright 2008 Google Inc. All rights reserved.
#
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file or at
# https://developers.google.com/open-source/licenses/bsd
"""Contains the Duration helper APIs."""
import datetime
from google.protobuf.duration_pb2 import Duration
def from_json_string(value: str) -> Duration:
"""Converts a string to Duration.
Args:
value: A string to be converted. The string must end with 's'. Any
fractional digits (or none) are accepted as long as they fit into
precision. For example: "1s", "1.01s", "1.0000001s", "-3.100s"
Raises:
ValueError: On parsing problems.
"""
duration = Duration()
duration.FromJsonString(value)
return duration
def from_microseconds(micros: float) -> Duration:
"""Converts microseconds to Duration."""
duration = Duration()
duration.FromMicroseconds(micros)
return duration
def from_milliseconds(millis: float) -> Duration:
"""Converts milliseconds to Duration."""
duration = Duration()
duration.FromMilliseconds(millis)
return duration
def from_nanoseconds(nanos: float) -> Duration:
"""Converts nanoseconds to Duration."""
duration = Duration()
duration.FromNanoseconds(nanos)
return duration
def from_seconds(seconds: float) -> Duration:
"""Converts seconds to Duration."""
duration = Duration()
duration.FromSeconds(seconds)
return duration
def from_timedelta(td: datetime.timedelta) -> Duration:
"""Converts timedelta to Duration."""
duration = Duration()
duration.FromTimedelta(td)
return duration
def to_json_string(duration: Duration) -> str:
"""Converts Duration to string format.
Returns:
A string converted from self. The string format will contains
3, 6, or 9 fractional digits depending on the precision required to
represent the exact Duration value. For example: "1s", "1.010s",
"1.000000100s", "-3.100s"
"""
return duration.ToJsonString()
def to_microseconds(duration: Duration) -> int:
"""Converts a Duration to microseconds."""
return duration.ToMicroseconds()
def to_milliseconds(duration: Duration) -> int:
"""Converts a Duration to milliseconds."""
return duration.ToMilliseconds()
def to_nanoseconds(duration: Duration) -> int:
"""Converts a Duration to nanoseconds."""
return duration.ToNanoseconds()
def to_seconds(duration: Duration) -> int:
"""Converts a Duration to seconds."""
return duration.ToSeconds()
def to_timedelta(duration: Duration) -> datetime.timedelta:
"""Converts Duration to timedelta."""
return duration.ToTimedelta()

View File

@@ -0,0 +1,37 @@
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler. DO NOT EDIT!
# NO CHECKED-IN PROTOBUF GENCODE
# source: google/protobuf/duration.proto
# Protobuf Python Version: 7.34.0
"""Generated protocol buffer code."""
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import runtime_version as _runtime_version
from google.protobuf import symbol_database as _symbol_database
from google.protobuf.internal import builder as _builder
_runtime_version.ValidateProtobufRuntimeVersion(
_runtime_version.Domain.PUBLIC,
7,
34,
0,
'',
'google/protobuf/duration.proto'
)
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1egoogle/protobuf/duration.proto\x12\x0fgoogle.protobuf\":\n\x08\x44uration\x12\x18\n\x07seconds\x18\x01 \x01(\x03R\x07seconds\x12\x14\n\x05nanos\x18\x02 \x01(\x05R\x05nanosB\x83\x01\n\x13\x63om.google.protobufB\rDurationProtoP\x01Z1google.golang.org/protobuf/types/known/durationpb\xf8\x01\x01\xa2\x02\x03GPB\xaa\x02\x1eGoogle.Protobuf.WellKnownTypesb\x06proto3')
_globals = globals()
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'google.protobuf.duration_pb2', _globals)
if not _descriptor._USE_C_DESCRIPTORS:
_globals['DESCRIPTOR']._loaded_options = None
_globals['DESCRIPTOR']._serialized_options = b'\n\023com.google.protobufB\rDurationProtoP\001Z1google.golang.org/protobuf/types/known/durationpb\370\001\001\242\002\003GPB\252\002\036Google.Protobuf.WellKnownTypes'
_globals['_DURATION']._serialized_start=51
_globals['_DURATION']._serialized_end=109
# @@protoc_insertion_point(module_scope)

View File

@@ -0,0 +1,37 @@
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler. DO NOT EDIT!
# NO CHECKED-IN PROTOBUF GENCODE
# source: google/protobuf/empty.proto
# Protobuf Python Version: 7.34.0
"""Generated protocol buffer code."""
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import runtime_version as _runtime_version
from google.protobuf import symbol_database as _symbol_database
from google.protobuf.internal import builder as _builder
_runtime_version.ValidateProtobufRuntimeVersion(
_runtime_version.Domain.PUBLIC,
7,
34,
0,
'',
'google/protobuf/empty.proto'
)
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1bgoogle/protobuf/empty.proto\x12\x0fgoogle.protobuf\"\x07\n\x05\x45mptyB}\n\x13\x63om.google.protobufB\nEmptyProtoP\x01Z.google.golang.org/protobuf/types/known/emptypb\xf8\x01\x01\xa2\x02\x03GPB\xaa\x02\x1eGoogle.Protobuf.WellKnownTypesb\x06proto3')
_globals = globals()
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'google.protobuf.empty_pb2', _globals)
if not _descriptor._USE_C_DESCRIPTORS:
_globals['DESCRIPTOR']._loaded_options = None
_globals['DESCRIPTOR']._serialized_options = b'\n\023com.google.protobufB\nEmptyProtoP\001Z.google.golang.org/protobuf/types/known/emptypb\370\001\001\242\002\003GPB\252\002\036Google.Protobuf.WellKnownTypes'
_globals['_EMPTY']._serialized_start=48
_globals['_EMPTY']._serialized_end=55
# @@protoc_insertion_point(module_scope)

View File

@@ -0,0 +1,37 @@
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler. DO NOT EDIT!
# NO CHECKED-IN PROTOBUF GENCODE
# source: google/protobuf/field_mask.proto
# Protobuf Python Version: 7.34.0
"""Generated protocol buffer code."""
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import runtime_version as _runtime_version
from google.protobuf import symbol_database as _symbol_database
from google.protobuf.internal import builder as _builder
_runtime_version.ValidateProtobufRuntimeVersion(
_runtime_version.Domain.PUBLIC,
7,
34,
0,
'',
'google/protobuf/field_mask.proto'
)
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n google/protobuf/field_mask.proto\x12\x0fgoogle.protobuf\"!\n\tFieldMask\x12\x14\n\x05paths\x18\x01 \x03(\tR\x05pathsB\x85\x01\n\x13\x63om.google.protobufB\x0e\x46ieldMaskProtoP\x01Z2google.golang.org/protobuf/types/known/fieldmaskpb\xf8\x01\x01\xa2\x02\x03GPB\xaa\x02\x1eGoogle.Protobuf.WellKnownTypesb\x06proto3')
_globals = globals()
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'google.protobuf.field_mask_pb2', _globals)
if not _descriptor._USE_C_DESCRIPTORS:
_globals['DESCRIPTOR']._loaded_options = None
_globals['DESCRIPTOR']._serialized_options = b'\n\023com.google.protobufB\016FieldMaskProtoP\001Z2google.golang.org/protobuf/types/known/fieldmaskpb\370\001\001\242\002\003GPB\252\002\036Google.Protobuf.WellKnownTypes'
_globals['_FIELDMASK']._serialized_start=53
_globals['_FIELDMASK']._serialized_end=86
# @@protoc_insertion_point(module_scope)

View File

@@ -0,0 +1,7 @@
# Protocol Buffers - Google's data interchange format
# Copyright 2008 Google Inc. All rights reserved.
#
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file or at
# https://developers.google.com/open-source/licenses/bsd

View File

@@ -0,0 +1,136 @@
# Protocol Buffers - Google's data interchange format
# Copyright 2008 Google Inc. All rights reserved.
#
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file or at
# https://developers.google.com/open-source/licenses/bsd
"""Determine which implementation of the protobuf API is used in this process.
"""
import importlib
import os
import sys
import warnings
_GOOGLE3_PYTHON_UPB_DEFAULT = True
def _ApiVersionToImplementationType(api_version):
if api_version == 2:
return 'cpp'
if api_version == 1:
raise ValueError('api_version=1 is no longer supported.')
if api_version == 0:
return 'python'
return None
_implementation_type = None
try:
# pylint: disable=g-import-not-at-top
from google.protobuf.internal import _api_implementation
# The compile-time constants in the _api_implementation module can be used to
# switch to a certain implementation of the Python API at build time.
_implementation_type = _ApiVersionToImplementationType(
_api_implementation.api_version)
except ImportError:
pass # Unspecified by compiler flags.
def _CanImport(mod_name):
try:
mod = importlib.import_module(mod_name)
# Work around a known issue in the classic bootstrap .par import hook.
if not mod:
raise ImportError(mod_name + ' import succeeded but was None')
return True
except ImportError:
return False
if _implementation_type is None:
if _CanImport('google._upb._message'):
_implementation_type = 'upb'
elif _CanImport('google.protobuf.pyext._message'):
_implementation_type = 'cpp'
else:
_implementation_type = 'python'
# This environment variable can be used to switch to a certain implementation
# of the Python API, overriding the compile-time constants in the
# _api_implementation module. Right now only 'python', 'cpp' and 'upb' are
# valid values. Any other value will raise error.
_implementation_type = os.getenv('PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION',
_implementation_type)
if _implementation_type not in ('python', 'cpp', 'upb'):
raise ValueError('PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION {0} is not '
'supported. Please set to \'python\', \'cpp\' or '
'\'upb\'.'.format(_implementation_type))
if 'PyPy' in sys.version and _implementation_type == 'cpp':
warnings.warn('PyPy does not work yet with cpp protocol buffers. '
'Falling back to the python implementation.')
_implementation_type = 'python'
_c_module = None
if _implementation_type == 'cpp':
try:
# pylint: disable=g-import-not-at-top
from google.protobuf.pyext import _message
sys.modules['google3.net.proto2.python.internal.cpp._message'] = _message
_c_module = _message
del _message
except ImportError:
# TODO: fail back to python
warnings.warn(
'Selected implementation cpp is not available.')
pass
if _implementation_type == 'upb':
try:
# pylint: disable=g-import-not-at-top
from google._upb import _message
_c_module = _message
del _message
except ImportError:
warnings.warn('Selected implementation upb is not available. '
'Falling back to the python implementation.')
_implementation_type = 'python'
pass
# Detect if serialization should be deterministic by default
try:
# The presence of this module in a build allows the proto implementation to
# be upgraded merely via build deps.
#
# NOTE: Merely importing this automatically enables deterministic proto
# serialization for C++ code, but we still need to export it as a boolean so
# that we can do the same for `_implementation_type == 'python'`.
#
# NOTE2: It is possible for C++ code to enable deterministic serialization by
# default _without_ affecting Python code, if the C++ implementation is not in
# use by this module. That is intended behavior, so we don't actually expose
# this boolean outside of this module.
#
# pylint: disable=g-import-not-at-top,unused-import
from google.protobuf import enable_deterministic_proto_serialization
_python_deterministic_proto_serialization = True
except ImportError:
_python_deterministic_proto_serialization = False
# Usage of this function is discouraged. Clients shouldn't care which
# implementation of the API is in use. Note that there is no guarantee
# that differences between APIs will be maintained.
# Please don't use this function if possible.
def Type():
return _implementation_type
# For internal use only
def IsPythonDefaultSerializationDeterministic():
return _python_deterministic_proto_serialization

View File

@@ -0,0 +1,122 @@
# Protocol Buffers - Google's data interchange format
# Copyright 2008 Google Inc. All rights reserved.
#
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file or at
# https://developers.google.com/open-source/licenses/bsd
"""Builds descriptors, message classes and services for generated _pb2.py.
This file is only called in python generated _pb2.py files. It builds
descriptors, message classes and services that users can directly use
in generated code.
"""
__author__ = 'jieluo@google.com (Jie Luo)'
from google.protobuf.internal import enum_type_wrapper
from google.protobuf.internal import python_message
from google.protobuf import message as _message
from google.protobuf import reflection as _reflection
from google.protobuf import symbol_database as _symbol_database
_sym_db = _symbol_database.Default()
def _BuildNestedDescriptors(module, msg_des, prefix):
for name, nested_msg in msg_des.nested_types_by_name.items():
module_name = prefix + name.upper()
module[module_name] = nested_msg
_BuildNestedDescriptors(module, nested_msg, module_name + '_')
for enum_des in msg_des.enum_types:
module[prefix + enum_des.name.upper()] = enum_des
def BuildMessageAndEnumDescriptors(file_des, module):
"""Builds message and enum descriptors.
Args:
file_des: FileDescriptor of the .proto file
module: Generated _pb2 module
"""
for (name, msg_des) in file_des.message_types_by_name.items():
module_name = '_' + name.upper()
module[module_name] = msg_des
_BuildNestedDescriptors(module, msg_des, module_name + '_')
def _BuildMessage(module_name, msg_des, prefix):
create_dict = {}
for name, nested_msg in msg_des.nested_types_by_name.items():
create_dict[name] = _BuildMessage(
module_name, nested_msg, prefix + msg_des.name + '.'
)
create_dict['DESCRIPTOR'] = msg_des
create_dict['__module__'] = module_name
create_dict['__qualname__'] = prefix + msg_des.name
message_class = _reflection.GeneratedProtocolMessageType(
msg_des.name, (_message.Message,), create_dict
)
_sym_db.RegisterMessage(message_class)
return message_class
def BuildTopDescriptorsAndMessages(file_des, module_name, module):
"""Builds top level descriptors and message classes.
Args:
file_des: FileDescriptor of the .proto file
module_name: str, the name of generated _pb2 module
module: Generated _pb2 module
"""
# top level enums
for (name, enum_des) in file_des.enum_types_by_name.items():
module['_' + name.upper()] = enum_des
module[name] = enum_type_wrapper.EnumTypeWrapper(enum_des)
for enum_value in enum_des.values:
module[enum_value.name] = enum_value.number
# top level extensions
for (name, extension_des) in file_des.extensions_by_name.items():
module[name.upper() + '_FIELD_NUMBER'] = extension_des.number
module[name] = extension_des
# services
for (name, service) in file_des.services_by_name.items():
module['_' + name.upper()] = service
# Build messages.
for (name, msg_des) in file_des.message_types_by_name.items():
module[name] = _BuildMessage(module_name, msg_des, '')
def AddHelpersToExtensions(file_des):
"""no-op to keep old generated code work with new runtime.
Args:
file_des: FileDescriptor of the .proto file
"""
# TODO: Remove this on-op
return
def BuildServices(file_des, module_name, module):
"""Builds services classes and services stub class.
Args:
file_des: FileDescriptor of the .proto file
module_name: str, the name of generated _pb2 module
module: Generated _pb2 module
"""
# pylint: disable=g-import-not-at-top
from google.protobuf import service_reflection
# pylint: enable=g-import-not-at-top
for (name, service) in file_des.services_by_name.items():
module[name] = service_reflection.GeneratedServiceType(
name, (),
dict(DESCRIPTOR=service, __module__=module_name))
stub_name = name + '_Stub'
module[stub_name] = service_reflection.GeneratedServiceStubType(
stub_name, (module[name],),
dict(DESCRIPTOR=service, __module__=module_name))

View File

@@ -0,0 +1,726 @@
# Protocol Buffers - Google's data interchange format
# Copyright 2008 Google Inc. All rights reserved.
#
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file or at
# https://developers.google.com/open-source/licenses/bsd
"""Contains container classes to represent different protocol buffer types.
This file defines container classes which represent categories of protocol
buffer field types which need extra maintenance. Currently these categories
are:
- Repeated scalar fields - These are all repeated fields which aren't
composite (e.g. they are of simple types like int32, string, etc).
- Repeated composite fields - Repeated fields which are composite. This
includes groups and nested messages.
"""
import collections.abc
import copy
import pickle
from typing import (
Any,
Iterable,
Iterator,
List,
MutableMapping,
MutableSequence,
NoReturn,
Optional,
Sequence,
TypeVar,
Union,
overload,
)
_T = TypeVar('_T')
_K = TypeVar('_K')
_V = TypeVar('_V')
from google.protobuf.descriptor import FieldDescriptor
class BaseContainer(Sequence[_T]):
"""Base container class."""
# Minimizes memory usage and disallows assignment to other attributes.
__slots__ = ['_message_listener', '_values']
def __init__(self, message_listener: Any) -> None:
"""
Args:
message_listener: A MessageListener implementation.
The RepeatedScalarFieldContainer will call this object's
Modified() method when it is modified.
"""
self._message_listener = message_listener
self._values = []
@overload
def __getitem__(self, key: int) -> _T:
...
@overload
def __getitem__(self, key: slice) -> List[_T]:
...
def __getitem__(self, key):
"""Retrieves item by the specified key."""
return self._values[key]
def __len__(self) -> int:
"""Returns the number of elements in the container."""
return len(self._values)
def __ne__(self, other: Any) -> bool:
"""Checks if another instance isn't equal to this one."""
# The concrete classes should define __eq__.
return not self == other
__hash__ = None
def __repr__(self) -> str:
return repr(self._values)
def sort(self, *args, **kwargs) -> None:
# Continue to support the old sort_function keyword argument.
# This is expected to be a rare occurrence, so use LBYL to avoid
# the overhead of actually catching KeyError.
if 'sort_function' in kwargs:
kwargs['cmp'] = kwargs.pop('sort_function')
self._values.sort(*args, **kwargs)
def reverse(self) -> None:
self._values.reverse()
# TODO: Remove this. BaseContainer does *not* conform to
# MutableSequence, only its subclasses do.
collections.abc.MutableSequence.register(BaseContainer)
class RepeatedScalarFieldContainer(BaseContainer[_T], MutableSequence[_T]):
"""Simple, type-checked, list-like container for holding repeated scalars."""
# Disallows assignment to other attributes.
__slots__ = ['_type_checker', '_field']
def __init__(
self,
message_listener: Any,
type_checker: Any,
field: Any = None,
) -> None:
"""Args:
message_listener: A MessageListener implementation. The
RepeatedScalarFieldContainer will call this object's Modified() method
when it is modified.
type_checker: A type_checkers.ValueChecker instance to run on elements
inserted into this container.
"""
super().__init__(message_listener)
self._type_checker = type_checker
self._field = field
def append(self, value: _T) -> None:
"""Appends an item to the list. Similar to list.append()."""
self._values.append(self._type_checker.CheckValue(value))
if not self._message_listener.dirty:
self._message_listener.Modified()
def insert(self, key: int, value: _T) -> None:
"""Inserts the item at the specified position. Similar to list.insert()."""
self._values.insert(key, self._type_checker.CheckValue(value))
if not self._message_listener.dirty:
self._message_listener.Modified()
def extend(self, elem_seq: Iterable[_T]) -> None:
"""Extends by appending the given iterable. Similar to list.extend()."""
elem_seq_iter = iter(elem_seq)
new_values = [self._type_checker.CheckValue(elem) for elem in elem_seq_iter]
if new_values:
self._values.extend(new_values)
self._message_listener.Modified()
def MergeFrom(
self,
other: Union['RepeatedScalarFieldContainer[_T]', Iterable[_T]],
) -> None:
"""Appends the contents of another repeated field of the same type to this
one. We do not check the types of the individual fields.
"""
self._values.extend(other)
self._message_listener.Modified()
def remove(self, elem: _T):
"""Removes an item from the list. Similar to list.remove()."""
self._values.remove(elem)
self._message_listener.Modified()
def pop(self, key: Optional[int] = -1) -> _T:
"""Removes and returns an item at a given index. Similar to list.pop()."""
value = self._values[key]
self.__delitem__(key)
return value
@overload
def __setitem__(self, key: int, value: _T) -> None:
...
@overload
def __setitem__(self, key: slice, value: Iterable[_T]) -> None:
...
def __setitem__(self, key, value) -> None:
"""Sets the item on the specified position."""
if isinstance(key, slice):
if key.step is not None:
raise ValueError('Extended slices not supported')
self._values[key] = map(self._type_checker.CheckValue, value)
self._message_listener.Modified()
else:
self._values[key] = self._type_checker.CheckValue(value)
self._message_listener.Modified()
def __delitem__(self, key: Union[int, slice]) -> None:
"""Deletes the item at the specified position."""
del self._values[key]
self._message_listener.Modified()
def __eq__(self, other: Any) -> bool:
"""Compares the current instance with another one."""
if self is other:
return True
# Special case for the same type which should be common and fast.
if isinstance(other, self.__class__):
return other._values == self._values
# We are presumably comparing against some other sequence type.
return other == self._values
def __deepcopy__(
self,
unused_memo: Any = None,
) -> 'RepeatedScalarFieldContainer[_T]':
clone = RepeatedScalarFieldContainer(
copy.deepcopy(self._message_listener), self._type_checker, self._field
)
clone.MergeFrom(self)
return clone
def __reduce__(self, **kwargs) -> NoReturn:
raise pickle.PickleError(
"Can't pickle repeated scalar fields, convert to list first")
def __array__(self, dtype=None, copy=None):
import numpy as np
if dtype is None:
cpp_type = self._field.cpp_type
if cpp_type == FieldDescriptor.CPPTYPE_INT32:
dtype = np.int32
elif cpp_type == FieldDescriptor.CPPTYPE_INT64:
dtype = np.int64
elif cpp_type == FieldDescriptor.CPPTYPE_UINT32:
dtype = np.uint32
elif cpp_type == FieldDescriptor.CPPTYPE_UINT64:
dtype = np.uint64
elif cpp_type == FieldDescriptor.CPPTYPE_DOUBLE:
dtype = np.float64
elif cpp_type == FieldDescriptor.CPPTYPE_FLOAT:
dtype = np.float32
elif cpp_type == FieldDescriptor.CPPTYPE_BOOL:
dtype = np.bool
elif cpp_type == FieldDescriptor.CPPTYPE_ENUM:
dtype = np.int32
elif self._field.type == FieldDescriptor.TYPE_BYTES:
dtype = 'S'
elif self._field.type == FieldDescriptor.TYPE_STRING:
dtype = str
else:
raise SystemError(
'Code should never reach here: message type detected in'
' RepeatedScalarFieldContainer'
)
return np.array(self._values, dtype=dtype, copy=True)
# TODO: Constrain T to be a subtype of Message.
class RepeatedCompositeFieldContainer(BaseContainer[_T], MutableSequence[_T]):
"""Simple, list-like container for holding repeated composite fields."""
# Disallows assignment to other attributes.
__slots__ = ['_message_descriptor']
def __init__(self, message_listener: Any, message_descriptor: Any) -> None:
"""
Note that we pass in a descriptor instead of the generated directly,
since at the time we construct a _RepeatedCompositeFieldContainer we
haven't yet necessarily initialized the type that will be contained in the
container.
Args:
message_listener: A MessageListener implementation.
The RepeatedCompositeFieldContainer will call this object's
Modified() method when it is modified.
message_descriptor: A Descriptor instance describing the protocol type
that should be present in this container. We'll use the
_concrete_class field of this descriptor when the client calls add().
"""
super().__init__(message_listener)
self._message_descriptor = message_descriptor
def add(self, **kwargs: Any) -> _T:
"""Adds a new element at the end of the list and returns it. Keyword
arguments may be used to initialize the element.
"""
new_element = self._message_descriptor._concrete_class(**kwargs)
new_element._SetListener(self._message_listener)
self._values.append(new_element)
if not self._message_listener.dirty:
self._message_listener.Modified()
return new_element
def append(self, value: _T) -> None:
"""Appends one element by copying the message."""
new_element = self._message_descriptor._concrete_class()
new_element._SetListener(self._message_listener)
new_element.CopyFrom(value)
self._values.append(new_element)
if not self._message_listener.dirty:
self._message_listener.Modified()
def insert(self, key: int, value: _T) -> None:
"""Inserts the item at the specified position by copying."""
new_element = self._message_descriptor._concrete_class()
new_element._SetListener(self._message_listener)
new_element.CopyFrom(value)
self._values.insert(key, new_element)
if not self._message_listener.dirty:
self._message_listener.Modified()
def extend(self, elem_seq: Iterable[_T]) -> None:
"""Extends by appending the given sequence of elements of the same type
as this one, copying each individual message.
"""
message_class = self._message_descriptor._concrete_class
listener = self._message_listener
values = self._values
for message in elem_seq:
new_element = message_class()
new_element._SetListener(listener)
new_element.MergeFrom(message)
values.append(new_element)
listener.Modified()
def MergeFrom(
self,
other: Union['RepeatedCompositeFieldContainer[_T]', Iterable[_T]],
) -> None:
"""Appends the contents of another repeated field of the same type to this
one, copying each individual message.
"""
self.extend(other)
def remove(self, elem: _T) -> None:
"""Removes an item from the list. Similar to list.remove()."""
self._values.remove(elem)
self._message_listener.Modified()
def pop(self, key: Optional[int] = -1) -> _T:
"""Removes and returns an item at a given index. Similar to list.pop()."""
value = self._values[key]
self.__delitem__(key)
return value
@overload
def __setitem__(self, key: int, value: _T) -> None:
...
@overload
def __setitem__(self, key: slice, value: Iterable[_T]) -> None:
...
def __setitem__(self, key, value):
# This method is implemented to make RepeatedCompositeFieldContainer
# structurally compatible with typing.MutableSequence. It is
# otherwise unsupported and will always raise an error.
raise TypeError(
f'{self.__class__.__name__} object does not support item assignment')
def __delitem__(self, key: Union[int, slice]) -> None:
"""Deletes the item at the specified position."""
del self._values[key]
self._message_listener.Modified()
def __eq__(self, other: Any) -> bool:
"""Compares the current instance with another one."""
if self is other:
return True
if not isinstance(other, self.__class__):
raise TypeError('Can only compare repeated composite fields against '
'other repeated composite fields.')
return self._values == other._values
class ScalarMap(MutableMapping[_K, _V]):
"""Simple, type-checked, dict-like container for holding repeated scalars."""
# Disallows assignment to other attributes.
__slots__ = ['_key_checker', '_value_checker', '_values', '_message_listener',
'_entry_descriptor']
def __init__(
self,
message_listener: Any,
key_checker: Any,
value_checker: Any,
entry_descriptor: Any,
) -> None:
"""
Args:
message_listener: A MessageListener implementation.
The ScalarMap will call this object's Modified() method when it
is modified.
key_checker: A type_checkers.ValueChecker instance to run on keys
inserted into this container.
value_checker: A type_checkers.ValueChecker instance to run on values
inserted into this container.
entry_descriptor: The MessageDescriptor of a map entry: key and value.
"""
self._message_listener = message_listener
self._key_checker = key_checker
self._value_checker = value_checker
self._entry_descriptor = entry_descriptor
self._values = {}
def __getitem__(self, key: _K) -> _V:
try:
return self._values[key]
except KeyError:
key = self._key_checker.CheckValue(key)
val = self._value_checker.DefaultValue()
self._values[key] = val
return val
def __contains__(self, item: _K) -> bool:
# We check the key's type to match the strong-typing flavor of the API.
# Also this makes it easier to match the behavior of the C++ implementation.
self._key_checker.CheckValue(item)
return item in self._values
@overload
def get(self, key: _K) -> Optional[_V]:
...
@overload
def get(self, key: _K, default: _T) -> Union[_V, _T]:
...
# We need to override this explicitly, because our defaultdict-like behavior
# will make the default implementation (from our base class) always insert
# the key.
def get(self, key, default=None):
if key in self:
return self[key]
else:
return default
def __setitem__(self, key: _K, value: _V) -> _T:
checked_key = self._key_checker.CheckValue(key)
checked_value = self._value_checker.CheckValue(value)
self._values[checked_key] = checked_value
self._message_listener.Modified()
def __delitem__(self, key: _K) -> None:
del self._values[key]
self._message_listener.Modified()
def __len__(self) -> int:
return len(self._values)
def __iter__(self) -> Iterator[_K]:
return iter(self._values)
def __repr__(self) -> str:
return repr(self._values)
def setdefault(self, key: _K, value: Optional[_V] = None) -> _V:
if value == None:
raise ValueError('The value for scalar map setdefault must be set.')
if key not in self._values:
self.__setitem__(key, value)
return self[key]
def MergeFrom(self, other: 'ScalarMap[_K, _V]') -> None:
self._values.update(other._values)
self._message_listener.Modified()
def InvalidateIterators(self) -> None:
# It appears that the only way to reliably invalidate iterators to
# self._values is to ensure that its size changes.
original = self._values
self._values = original.copy()
original[None] = None
# This is defined in the abstract base, but we can do it much more cheaply.
def clear(self) -> None:
self._values.clear()
self._message_listener.Modified()
def GetEntryClass(self) -> Any:
return self._entry_descriptor._concrete_class
class MessageMap(MutableMapping[_K, _V]):
"""Simple, type-checked, dict-like container for with submessage values."""
# Disallows assignment to other attributes.
__slots__ = ['_key_checker', '_values', '_message_listener',
'_message_descriptor', '_entry_descriptor']
def __init__(
self,
message_listener: Any,
message_descriptor: Any,
key_checker: Any,
entry_descriptor: Any,
) -> None:
"""
Args:
message_listener: A MessageListener implementation.
The ScalarMap will call this object's Modified() method when it
is modified.
key_checker: A type_checkers.ValueChecker instance to run on keys
inserted into this container.
value_checker: A type_checkers.ValueChecker instance to run on values
inserted into this container.
entry_descriptor: The MessageDescriptor of a map entry: key and value.
"""
self._message_listener = message_listener
self._message_descriptor = message_descriptor
self._key_checker = key_checker
self._entry_descriptor = entry_descriptor
self._values = {}
def __getitem__(self, key: _K) -> _V:
key = self._key_checker.CheckValue(key)
try:
return self._values[key]
except KeyError:
new_element = self._message_descriptor._concrete_class()
new_element._SetListener(self._message_listener)
self._values[key] = new_element
self._message_listener.Modified()
return new_element
def get_or_create(self, key: _K) -> _V:
"""get_or_create() is an alias for getitem (ie. map[key]).
Args:
key: The key to get or create in the map.
This is useful in cases where you want to be explicit that the call is
mutating the map. This can avoid lint errors for statements like this
that otherwise would appear to be pointless statements:
msg.my_map[key]
"""
return self[key]
@overload
def get(self, key: _K) -> Optional[_V]:
...
@overload
def get(self, key: _K, default: _T) -> Union[_V, _T]:
...
# We need to override this explicitly, because our defaultdict-like behavior
# will make the default implementation (from our base class) always insert
# the key.
def get(self, key, default=None):
if key in self:
return self[key]
else:
return default
def __contains__(self, item: _K) -> bool:
item = self._key_checker.CheckValue(item)
return item in self._values
def __setitem__(self, key: _K, value: _V) -> NoReturn:
raise ValueError('May not set values directly, call my_map[key].foo = 5')
def __delitem__(self, key: _K) -> None:
key = self._key_checker.CheckValue(key)
del self._values[key]
self._message_listener.Modified()
def __len__(self) -> int:
return len(self._values)
def __iter__(self) -> Iterator[_K]:
return iter(self._values)
def __repr__(self) -> str:
return repr(self._values)
def setdefault(self, key: _K, value: Optional[_V] = None) -> _V:
raise NotImplementedError(
'Set message map value directly is not supported, call'
' my_map[key].foo = 5'
)
def MergeFrom(self, other: 'MessageMap[_K, _V]') -> None:
# pylint: disable=protected-access
for key in other._values:
# According to documentation: "When parsing from the wire or when merging,
# if there are duplicate map keys the last key seen is used".
if key in self:
del self[key]
self[key].CopyFrom(other[key])
# self._message_listener.Modified() not required here, because
# mutations to submessages already propagate.
def InvalidateIterators(self) -> None:
# It appears that the only way to reliably invalidate iterators to
# self._values is to ensure that its size changes.
original = self._values
self._values = original.copy()
original[None] = None
# This is defined in the abstract base, but we can do it much more cheaply.
def clear(self) -> None:
self._values.clear()
self._message_listener.Modified()
def GetEntryClass(self) -> Any:
return self._entry_descriptor._concrete_class
class _UnknownField:
"""A parsed unknown field."""
# Disallows assignment to other attributes.
__slots__ = ['_field_number', '_wire_type', '_data']
def __init__(self, field_number, wire_type, data):
self._field_number = field_number
self._wire_type = wire_type
self._data = data
return
def __lt__(self, other):
# pylint: disable=protected-access
return self._field_number < other._field_number
def __eq__(self, other):
if self is other:
return True
# pylint: disable=protected-access
return (self._field_number == other._field_number and
self._wire_type == other._wire_type and
self._data == other._data)
class UnknownFieldRef: # pylint: disable=missing-class-docstring
def __init__(self, parent, index):
self._parent = parent
self._index = index
def _check_valid(self):
if not self._parent:
raise ValueError('UnknownField does not exist. '
'The parent message might be cleared.')
if self._index >= len(self._parent):
raise ValueError('UnknownField does not exist. '
'The parent message might be cleared.')
@property
def field_number(self):
self._check_valid()
# pylint: disable=protected-access
return self._parent._internal_get(self._index)._field_number
@property
def wire_type(self):
self._check_valid()
# pylint: disable=protected-access
return self._parent._internal_get(self._index)._wire_type
@property
def data(self):
self._check_valid()
# pylint: disable=protected-access
return self._parent._internal_get(self._index)._data
class UnknownFieldSet:
"""UnknownField container"""
# Disallows assignment to other attributes.
__slots__ = ['_values']
def __init__(self):
self._values = []
def __getitem__(self, index):
if self._values is None:
raise ValueError('UnknownFields does not exist. '
'The parent message might be cleared.')
size = len(self._values)
if index < 0:
index += size
if index < 0 or index >= size:
raise IndexError('index %d out of range'.index)
return UnknownFieldRef(self, index)
def _internal_get(self, index):
return self._values[index]
def __len__(self):
if self._values is None:
raise ValueError('UnknownFields does not exist. '
'The parent message might be cleared.')
return len(self._values)
def _add(self, field_number, wire_type, data):
unknown_field = _UnknownField(field_number, wire_type, data)
self._values.append(unknown_field)
return unknown_field
def __iter__(self):
for i in range(len(self)):
yield UnknownFieldRef(self, i)
def _extend(self, other):
if other is None:
return
# pylint: disable=protected-access
self._values.extend(other._values)
def __eq__(self, other):
if self is other:
return True
# Sort unknown fields because their order shouldn't
# affect equality test.
values = list(self._values)
if other is None:
return not values
values.sort()
# pylint: disable=protected-access
other_values = sorted(other._values)
return values == other_values
def _clear(self):
for value in self._values:
# pylint: disable=protected-access
if isinstance(value._data, UnknownFieldSet):
value._data._clear() # pylint: disable=protected-access
self._values = None

Some files were not shown because too many files have changed in this diff Show More