diff --git a/.flake8 b/.flake8 index ed4bb847..01931aba 100644 --- a/.flake8 +++ b/.flake8 @@ -12,4 +12,6 @@ exclude = # This contains our built documentation build, # This contains builds of flake8 that we don't want to check - dist \ No newline at end of file + dist + # This folder contains auto generated files which doesnt make sense to check + generated \ No newline at end of file diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..78b0b426 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,38 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates + +version: 2 +updates: + - package-ecosystem: "pip" + directory: "./02_mqtt-cluster/requirements.txt" # Location of package manifests + schedule: + interval: "weekly" + allow: + # Allow both direct and indirect updates for all packages + - dependency-type: "all" + + - package-ecosystem: "pip" + directory: "./03_uns_graphdb/requirements.txt" # Location of package manifests + schedule: + interval: "weekly" + allow: + # Allow both direct and indirect updates for all packages + - dependency-type: "all" + + - package-ecosystem: "pip" + directory: "./04_uns_historian/requirements.txt" # Location of package manifests + schedule: + interval: "weekly" + allow: + # Allow both direct and indirect updates for all packages + - dependency-type: "all" + + - package-ecosystem: "pip" + directory: "./05_sparkplugb/requirements.txt" # Location of package manifests + schedule: + interval: "weekly" + allow: + # Allow both direct and indirect updates for all packages + - dependency-type: "all" diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 0b3ec91e..d8665381 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -5,42 +5,57 @@ name: UNS Project on: push: - branches: [ main ] + branches: [main] pull_request: - branches: [ main ] + branches: [main] permissions: contents: read jobs: build: - runs-on: ubuntu-latest + environment: dev steps: - - uses: actions/checkout@v3 - - name: Set up Python 3.10 - uses: actions/setup-python@v3 - with: - python-version: "3.10" - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install flake8 pytest - if [ -f ./02_mqtt-cluster/requirements.txt ]; then pip install -r ./02_mqtt-cluster/requirements.txt; fi - if [ -f ./02_mqtt-cluster/requirements_dev.txt ]; then pip install -r ./02_mqtt-cluster/requirements_dev.txt; fi - if [ -f ./03_uns_graphdb/requirements.txt ]; then pip install -r ./03_uns_graphdb/requirements.txt; fi - if [ -f ./03_uns_graphdb/requirements_dev.txt ]; then pip install -r ./03_uns_graphdb/requirements_dev.txt; fi - if [ -f ./04_uns_historian/requirements.txt ]; then pip install -r ./04_uns_historian/requirements.txt; fi - if [ -f ./04_uns_historian/requirements_dev.txt ]; then pip install -r ./04_uns_historian/requirements_dev.txt; fi - - name: Lint with flake8 - run: | - # stop the build if there are Python syntax errors or undefined names - flake8 ./02_mqtt-cluster/src ./02_mqtt-cluster/test ./03_uns_graphdb/src ./03_uns_graphdb/test ./04_uns_historian/src ./04_uns_historian/test --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 ./02_mqtt-cluster/src ./02_mqtt-cluster/test ./03_uns_graphdb/src ./03_uns_graphdb/test ./04_uns_historian/src ./04_uns_historian/test --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - name: Test with pytest - run: | - pytest -m "not integrationtest" ./02_mqtt-cluster/test/ - pytest -m "not integrationtest" ./03_uns_graphdb/test/ - pytest -m "not integrationtest" ./04_uns_historian/test/ \ No newline at end of file + - uses: actions/checkout@v3 + - name: Set up Python 3.10 + uses: actions/setup-python@v3 + with: + python-version: "3.10" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install flake8 pytest + if [ -f ./02_mqtt-cluster/requirements.txt ]; then pip install --upgrade -r ./02_mqtt-cluster/requirements.txt; fi + if [ -f ./02_mqtt-cluster/requirements_dev.txt ]; then pip install --upgrade -r ./02_mqtt-cluster/requirements_dev.txt; fi + if [ -f ./03_uns_graphdb/requirements.txt ]; then pip install --upgrade -r ./03_uns_graphdb/requirements.txt; fi + if [ -f ./03_uns_graphdb/requirements_dev.txt ]; then pip install --upgrade -r ./03_uns_graphdb/requirements_dev.txt; fi + if [ -f ./04_uns_historian/requirements.txt ]; then pip install --upgrade -r ./04_uns_historian/requirements.txt; fi + if [ -f ./04_uns_historian/requirements_dev.txt ]; then pip install --upgrade -r ./04_uns_historian/requirements_dev.txt; fi + if [ -f ./05_sparkplugb/requirements.txt ]; then pip install --upgrade -r ./05_sparkplugb/requirements.txt; fi + if [ -f ./05_sparkplugb/requirements_dev.txt ]; then pip install --upgrade -r ./05_sparkplugb/requirements_dev.txt; fi + + - name: Lint with flake8 + run: | + # stop the build if there are Python syntax errors or undefined names + flake8 ./02_mqtt-cluster/src ./02_mqtt-cluster/test ./03_uns_graphdb/src ./03_uns_graphdb/test ./04_uns_historian/src ./04_uns_historian/test ./05_sparkplugb/src ./05_sparkplugb/test --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 ./02_mqtt-cluster/src ./02_mqtt-cluster/test ./03_uns_graphdb/src ./03_uns_graphdb/test ./04_uns_historian/src ./04_uns_historian/test ./05_sparkplugb/src ./05_sparkplugb/test --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + + - name: Test with pytest + env: + UNS_mqtt.host: ${{ secrets.MQTT_HOST }} + UNS_mqtt.port: ${{ secrets.MQTT_PORT }} + UNS_mqtt.username: ${{ secrets.MQTT_USER }} + UNS_mqtt.password: ${{ secrets.MQTT_PWD }} + UNS_graphdb.url: ${{ secrets.NEO4J_URL }} + UNS_graphdb.username: ${{ secrets.NEO4J_USER }} + UNS_graphdb.password: ${{ secrets.NEO4J_PWD }} + + run: | + pytest -m "not integrationtest" ./02_mqtt-cluster/test/ + pytest -m "not integrationtest" ./03_uns_graphdb/test/ + pytest -m "not integrationtest" ./04_uns_historian/test/ + pytest -m "not integrationtest" ./05_sparkplugb/test/ diff --git a/.github/workflows/uns_graphdb-app.yml b/.github/workflows/uns_graphdb-app.yml index c6ced528..d1d91ec1 100644 --- a/.github/workflows/uns_graphdb-app.yml +++ b/.github/workflows/uns_graphdb-app.yml @@ -5,45 +5,53 @@ name: UNS Graph DB MQTT Client on: push: - branches: - - "**" + branches: + - "**" paths: - "02_mqtt-cluster/**" - "03_uns_graphdb/**" pull_request: - branches: + branches: - "**" paths: - "02_mqtt-cluster/**" - - "03_uns_graphdb/**" + - "03_uns_graphdb/**" permissions: contents: read jobs: build: - runs-on: ubuntu-latest + environment: dev steps: - - uses: actions/checkout@v3 - - name: Set up Python 3.10 - uses: actions/setup-python@v3 - with: - python-version: "3.10" - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install flake8 pytest - if [ -f ./03_uns_graphdb/requirements.txt ]; then pip install -r ./03_uns_graphdb/requirements.txt; fi - if [ -f ./03_uns_graphdb/requirements_dev.txt ]; then pip install -r ./03_uns_graphdb/requirements_dev.txt; fi + - uses: actions/checkout@v3 + - name: Set up Python 3.10 + uses: actions/setup-python@v3 + with: + python-version: "3.10" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install flake8 pytest + if [ -f ./03_uns_graphdb/requirements.txt ]; then pip install --upgrade -r ./03_uns_graphdb/requirements.txt; fi + if [ -f ./03_uns_graphdb/requirements_dev.txt ]; then pip install --upgrade -r ./03_uns_graphdb/requirements_dev.txt; fi + + - name: Lint with flake8 + run: | + # stop the build if there are Python syntax errors or undefined names + flake8 ./03_uns_graphdb/src ./03_uns_graphdb/test --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 ./03_uns_graphdb/src ./03_uns_graphdb/test --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - name: Lint with flake8 - run: | - # stop the build if there are Python syntax errors or undefined names - flake8 ./03_uns_graphdb/src ./03_uns_graphdb/test --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 ./03_uns_graphdb/src ./03_uns_graphdb/test --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - - name: Test with pytest - run: | - pytest -m "not integrationtest" ./03_uns_graphdb/test/ + - name: Test with pytest + env: + UNS_mqtt.host: ${{ secrets.MQTT_HOST }} + UNS_mqtt.port: ${{ secrets.MQTT_PORT }} + UNS_mqtt.username: ${{ secrets.MQTT_USER }} + UNS_mqtt.password: ${{ secrets.MQTT_PWD }} + UNS_graphdb.url: ${{ secrets.NEO4J_URL }} + UNS_graphdb.username: ${{ secrets.NEO4J_USER }} + UNS_graphdb.password: ${{ secrets.NEO4J_PWD }} + run: | + pytest -m "not integrationtest" ./03_uns_graphdb/test/ diff --git a/.github/workflows/uns_historian-app.yml b/.github/workflows/uns_historian-app.yml index 5c11d419..4d9d9a33 100644 --- a/.github/workflows/uns_historian-app.yml +++ b/.github/workflows/uns_historian-app.yml @@ -5,45 +5,44 @@ name: UNS Historian MQTT Client on: push: - branches: + branches: - "**" paths: - "02_mqtt-cluster/**" - "04_uns_historian/**" pull_request: - branches: + branches: - "**" paths: - "02_mqtt-cluster/**" - - "04_uns_historian/**" + - "04_uns_historian/**" permissions: contents: read jobs: build: - runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - name: Set up Python 3.10 - uses: actions/setup-python@v3 - with: - python-version: "3.10" - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install flake8 pytest - if [ -f ./04_uns_historian/requirements.txt ]; then pip install -r ./04_uns_historian/requirements.txt; fi - if [ -f ./04_uns_historian/requirements_dev.txt ]; then pip install -r ./04_uns_historian/requirements_dev.txt; fi + - uses: actions/checkout@v3 + - name: Set up Python 3.10 + uses: actions/setup-python@v3 + with: + python-version: "3.10" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install flake8 pytest + if [ -f ./04_uns_historian/requirements.txt ]; then pip install --upgrade -r ./04_uns_historian/requirements.txt; fi + if [ -f ./04_uns_historian/requirements_dev.txt ]; then pip install --upgrade -r ./04_uns_historian/requirements_dev.txt; fi + + - name: Lint with flake8 + run: | + # stop the build if there are Python syntax errors or undefined names + flake8 ./04_uns_historian/src ./04_uns_historian/test --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 ./04_uns_historian/src ./04_uns_historian/test --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - name: Lint with flake8 - run: | - # stop the build if there are Python syntax errors or undefined names - flake8 ./04_uns_historian/src ./04_uns_historian/test --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 ./04_uns_historian/src ./04_uns_historian/test --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - - name: Test with pytest - run: | - pytest -m "not integrationtest" ./04_uns_historian/test/ + - name: Test with pytest + run: | + pytest -m "not integrationtest" ./04_uns_historian/test/ diff --git a/.github/workflows/uns_sparkplugb-app.yml b/.github/workflows/uns_sparkplugb-app.yml new file mode 100644 index 00000000..c9bd4d38 --- /dev/null +++ b/.github/workflows/uns_sparkplugb-app.yml @@ -0,0 +1,48 @@ +# This workflow will install Python dependencies, run tests and lint with a single version of Python +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: UNS Sparkplug B Decoder + +on: + push: + branches: + - "**" + paths: + - "02_mqtt-cluster/**" + - "05_sparkplugb/**" + pull_request: + branches: + - "**" + paths: + - "02_mqtt-cluster/**" + - "05_sparkplugb/**" +permissions: + contents: read + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - name: Set up Python 3.10 + uses: actions/setup-python@v3 + with: + python-version: "3.10" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install flake8 pytest + if [ -f ./05_sparkplugb/requirements.txt ]; then pip install --upgrade -r ./05_sparkplugb/requirements.txt; fi + if [ -f ./05_sparkplugb/requirements_dev.txt ]; then pip install --upgrade -r ./05_sparkplugb/requirements_dev.txt; fi + + - name: Lint with flake8 + run: | + # stop the build if there are Python syntax errors or undefined names + flake8 ./05_sparkplugb/src ./05_sparkplugb/test --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 ./05_sparkplugb/src ./05_sparkplugb/test --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + + - name: Test with pytest + run: | + pytest -m "not integrationtest" ./05_sparkplugb/test/ diff --git a/.vscode/settings.json b/.vscode/settings.json index a5746d90..dc706931 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,16 +1,44 @@ { "cSpell.words": [ + "aarch", + "DBIRTH", + "DCMD", + "DDATA", + "DDEATH", "EMQX", "graphdb", "HIVEMQ", "hypertable", "IIOT", + "integrationtest", "k8scluster", + "keepalive", + "Kubernetes", "microk8s", + "MOSQUITTO", + "mqtt", + "NBIRTH", + "NCMD", + "NDATA", + "NDEATH", + "packettypes", "paho", + "ppcle", + "propertyset", + "propertysets", + "proto", + "protobuf", + "pytest", + "SCADA", + "sparkplugb", + "sslmode", + "Tahu", "timescaledb", + "unspublisher", + "userdata", "venv", - "VERNEMQ" + "VERNEMQ", + "virtualenv" ], "python.formatting.provider": "yapf", "python.testing.unittestEnabled": false, @@ -18,7 +46,8 @@ "python.testing.pytestArgs": [ "./02_mqtt-cluster/test", "./03_uns_graphdb/test", - "./04_uns_historian/test" + "./04_uns_historian/test", + "./05_sparkplugb/test" ], "python.analysis.extraPaths": [ "./02_mqtt-cluster/src", @@ -30,7 +59,11 @@ "./04_uns_historian/src", "./04_uns_historian/src/uns_historian", "./04_uns_historian/test", + "./05_sparkplugb/src", + "./05_sparkplugb/src/uns_sparkplugb", + "./05_sparkplugb/test", ], "python.linting.flake8Enabled": true, - "python.linting.enabled": true + "python.linting.enabled": true, + "python.analysis.typeCheckingMode": "off" } \ No newline at end of file diff --git a/02_mqtt-cluster/requirements.txt b/02_mqtt-cluster/requirements.txt index 382e7554..dd51144e 100644 --- a/02_mqtt-cluster/requirements.txt +++ b/02_mqtt-cluster/requirements.txt @@ -1,3 +1,3 @@ -wheel +wheel>=0.37.1 paho-mqtt==1.6.1 -logger +logger>=1.4 diff --git a/02_mqtt-cluster/setup.cfg b/02_mqtt-cluster/setup.cfg new file mode 100644 index 00000000..491801f6 --- /dev/null +++ b/02_mqtt-cluster/setup.cfg @@ -0,0 +1,21 @@ +[metadata] +name = uns_mqtt + +author = Ashwin Krishnan +author_email = mkashwin@gmail.com +description = Base package for mqtt clients required for all UNS clients to be built on MQTT +long_description = file: README.md +long_description_content_type = text/markdown +classifiers = + Programming Language :: Python :: 3 + License :: OSI Approved :: MIT License + Operating System :: OS Independent + +[options] +package_dir = + = src +packages = find: +python_requires = >=3.10.5 + +[options.packages.find] +where = src \ No newline at end of file diff --git a/02_mqtt-cluster/src/uns_mqtt/mqtt_listener.py b/02_mqtt-cluster/src/uns_mqtt/mqtt_listener.py index 9d758d73..3e00bf33 100644 --- a/02_mqtt-cluster/src/uns_mqtt/mqtt_listener.py +++ b/02_mqtt-cluster/src/uns_mqtt/mqtt_listener.py @@ -14,7 +14,7 @@ class Uns_MQTT_ClientWrapper(mqtt_client.Client): """ - Wrapper over te paho.mqtt.client to implement most commont MQTT related functionality + Wrapper over te paho.mqtt.client to implement most common MQTT related functionality The call only needs to implement the callback function on_message """ MQTTv5 = mqtt_client.MQTTv5 @@ -66,7 +66,7 @@ def __init__(self, self.clean_session = clean_session if (protocol == mqtt_client.MQTTv5): - # if MQTT version is v5.0 the ignore cleansession in the constructor + # if MQTT version is v5.0 the ignore clean_session in the constructor clean_session = None super().__init__(client_id, clean_session, userdata, protocol, @@ -234,8 +234,8 @@ def isTopicMatching(topicWithWildcard: str, topic: str) -> bool: if (topicWithWildcard is not None): regexList = topicWithWildcard.split('/') # Using Regex to do matching - # replace all occurrences of "+" wildcard with [^/]* -> any set of charecters except "/" - # replace all occurrences of "#" wildcard with (.)* -> any set of charecters including "/" + # replace all occurrences of "+" wildcard with [^/]* -> any set of characters except "/" + # replace all occurrences of "#" wildcard with (.)* -> any set of characters including "/" regexExp = "" for value in regexList: if (value == "+"): diff --git a/02_mqtt-cluster/test/test_mqtt_listner.py b/02_mqtt-cluster/test/test_mqtt_listner.py index 5a6ec317..7902f641 100644 --- a/02_mqtt-cluster/test/test_mqtt_listner.py +++ b/02_mqtt-cluster/test/test_mqtt_listner.py @@ -33,7 +33,7 @@ @pytest.mark.parametrize("protocol", [(Uns_MQTT_ClientWrapper.MQTTv5), (Uns_MQTT_ClientWrapper.MQTTv311)]) # (UNS_MQTT_Listener.MQTTv31)]) -# There appears to be a bug for MQTTv31. The call backs are not occuring +# There appears to be a bug for MQTTv31. The call backs are not occurring @pytest.mark.parametrize("transport,port,tls", [("tcp", 1883, None), ("websockets", 8083, None), ("tcp", 8883, { @@ -104,7 +104,6 @@ def on_connect(client, userdata, flags, rc, properties=None): @pytest.mark.parametrize("protocol", [(Uns_MQTT_ClientWrapper.MQTTv5), (Uns_MQTT_ClientWrapper.MQTTv311), (Uns_MQTT_ClientWrapper.MQTTv31)]) -# There appears to be a bug for MQTTv31. The call backs are not occuring @pytest.mark.parametrize("transport,port,tls", [("tcp", 1884, None), ("websockets", 8090, None), ("tcp", 8885, { @@ -277,6 +276,13 @@ def test_del_key_from_dict(message: dict, ignored_attr: list, "timestamp": 123456, "val1": 1234 }), + ("topic1", { + "timestamp": 123456, + "val1": 1234 + }, {}, { + "timestamp": 123456, + "val1": 1234 + }), ("topic1", { "timestamp": 123456, "val1": 1234 diff --git a/03_uns_graphdb/Readme.md b/03_uns_graphdb/Readme.md index 1d67e22f..dde69b89 100644 --- a/03_uns_graphdb/Readme.md +++ b/03_uns_graphdb/Readme.md @@ -108,6 +108,9 @@ The set of test for this module is executed by ```python source env_graphdb/bin/activate python -m pip install -r requirements_dev.txt +#run all tests excluding integration tests +pytest -m "not integrationtest" test/ +# runs all tests pytest test/ ``` @@ -148,19 +151,18 @@ will result in a node in the GraphDB (erp:enterprise : { id1: "identifier", sensor1: 100, value1: 202203011145}) ``` ### Examples of MQTT Message being persisted to the GraphDB -- By publishing the following message to the MQTT Broker +- By publishing the following message to the MQTT Broker
![MQTT Message to ](../images/Message_2_GraphDBMQTTClient.png) -- We get the following graph created +- We get the following graph created
![Graph View](../images/GraphDB_view.png) - Each level of the topic is represented as a node with a relationship also established between the nodes. The label to the nodes is also assigned as per the Node Types which were [configured](#key-configurations-to-provide). e.g. the node `my_device` in this message has the label `DEVICE` in the GraphDB. - e.g. the node `area51` in this message has the label `AREA` in the GraphDB. + e.g. the node `area51` in this message has the label `AREA` in the GraphDB.
![Graph View](../images/GraphDB_view0.png) -- A Tabular of the same data - +- A Tabular of the same data
![Graph View](../images/GraphDB_Textview.png) ## Limitations / workarounds @@ -168,7 +170,7 @@ will result in a node in the GraphDB Neo4j does not support nested attributes. If your message contains nested data the current logic will flatten the JSON object. See the function [graphdb_handler.py#_flatten_json_for_Neo4J()](./src/uns_graphdb/graphdb_handler.py#_flatten_json_for_Neo4J) -* [x] Handling exceptional case of mqtt message containg the key ***"node_name"***. +* [x] Handling exceptional case of mqtt message containing the key ***"node_name"***. If your MQTT message contains the key ***"node_name"***, The key will be changed to uppercase before storing. This is because our application uses the key ***"node_name"*** to uniquely identify the node. This is the stripped topic name. The logic of this is in the function [graphdb_handler.py#_flatten_json_for_Neo4J()](./src/uns_graphdb/graphdb_handler.py#_flatten_json_for_Neo4J) * [ ] Current code & configurations have not yet considered securing the database and encrypted connections diff --git a/03_uns_graphdb/conf/settings.yaml b/03_uns_graphdb/conf/settings.yaml index 5c453b10..dbd00bd8 100644 --- a/03_uns_graphdb/conf/settings.yaml +++ b/03_uns_graphdb/conf/settings.yaml @@ -1,26 +1,24 @@ mqtt: host: "broker.emqx.io" # must be specified. No defaults e.g.198.168.220.3 broker.emqx.io port: 1883 # Default value 1883 if not specified - topic : "test/uns/#" # Default Topic is a wild card #. if required can be tuned to listen subset + topic: "test/uns/#" # Default Topic is a wild card #. if required can be tuned to listen subset qos: 1 # Default value is 1. Recommend 1 or 2. Do not use 0 keep_alive: 60 - reconnect_on_failure : True - version : 5 # Default is MQTTv5. Valid values are : 5 (for MQTTv5), 4 (for MQTTv311) , 3(for MQTTv31) - #clean_session: false # specify this only if the protocol is not MQTTv5 - transport : "tcp" # Default is tcp. Valid values are: websockets, tcp - ignored_attributes : {'+': "timestamp"} # dict of topic , list of attributes which are to be ignored from persistance. - # e.g. {'topic1' : ["attr1", "attr2", "attr2.subAttr1" ], - # 'topic2/+' : ["A", "A.B.C"], - # 'topic3/#' : ["x", "Y"]} - timestamp_attribute : "timestamp" # the attribute name which should contain the timestamp of the message's publishing - - + reconnect_on_failure: True + version: 5 # Default is MQTTv5. Valid values are : 5 (for MQTTv5), 4 (for MQTTv311) , 3(for MQTTv31) + #clean_session: false # specify this only if the protocol is not MQTTv5 + transport: "tcp" # Default is tcp. Valid values are: websockets, tcp + ignored_attributes: + {} # dict of topic , list of attributes which are to be ignored from persistance. + # e.g. {'topic1' : ["attr1", "attr2", "attr2.subAttr1" ], + # 'topic2/+' : ["A", "A.B.C"], + # 'topic3/#' : ["x", "Y"]} + timestamp_attribute: "timestamp" # the attribute name which should contain the timestamp of the message's publishing graphdb: - url: 'bolt://localhost:7687' -# database: '' + url: "bolt://localhost:7687" + # url: "neo4j+s://4da22061.databases.neo4j.io" + # database: "Instance01" node_types: ["ENTERPRISE", "FACILITY", "AREA", "LINE", "DEVICE"] # Based on ISA-95 part 2 the nested depth - ignored_attributes : {} # attributes in the message to ignore dynaconf_merge: true - \ No newline at end of file diff --git a/03_uns_graphdb/requirements.txt b/03_uns_graphdb/requirements.txt index c92c4924..6dc8b50b 100644 --- a/03_uns_graphdb/requirements.txt +++ b/03_uns_graphdb/requirements.txt @@ -1,5 +1,5 @@ -wheel -neo4j>=4.4.1 -paho-mqtt==1.6.1 -dynaconf -logger \ No newline at end of file +wheel>=0.37.1 +neo4j>=4.4.4 +paho-mqtt>=1.6.1 +dynaconf>=3.1.9 +logger>=1.4 \ No newline at end of file diff --git a/03_uns_graphdb/setup.cfg b/03_uns_graphdb/setup.cfg new file mode 100644 index 00000000..9b25c053 --- /dev/null +++ b/03_uns_graphdb/setup.cfg @@ -0,0 +1,21 @@ +[metadata] +name = uns_graphdb + +author = Ashwin Krishnan +author_email = mkashwin@gmail.com +description = Graph Database implementation for a UNS setup +long_description = file: README.md +long_description_content_type = text/markdown +classifiers = + Programming Language :: Python :: 3 + License :: OSI Approved :: MIT License + Operating System :: OS Independent + +[options] +package_dir = + = src +packages = find: +python_requires = >=3.10.5 + +[options.packages.find] +where = src \ No newline at end of file diff --git a/03_uns_graphdb/src/uns_graphdb/graphdb_handler.py b/03_uns_graphdb/src/uns_graphdb/graphdb_handler.py index a2cabcb7..2a5d8e4a 100644 --- a/03_uns_graphdb/src/uns_graphdb/graphdb_handler.py +++ b/03_uns_graphdb/src/uns_graphdb/graphdb_handler.py @@ -75,7 +75,7 @@ def connect(self, retry: int = 0): raise ex else: retry += 1 - LOGGER.error("Error Connecting to %s. Error: %s", + LOGGER.error("Error Connecting to %s.\n Error: %s", self.database, str(ex), stack_info=True, @@ -141,7 +141,7 @@ def persistMQTTmsg(self, else: retry += 1 LOGGER.error( - "Error persisting \ntopic:%s \nmessage %s. on Error:", + "Error persisting \ntopic:%s \nmessage %s. on Error: %s", topic, str(message), str(ex), diff --git a/03_uns_graphdb/test/test_graphdb_config.py b/03_uns_graphdb/test/test_graphdb_config.py index a6df1a21..dc12610e 100644 --- a/03_uns_graphdb/test/test_graphdb_config.py +++ b/03_uns_graphdb/test/test_graphdb_config.py @@ -88,7 +88,7 @@ def test_mqtt_config(): ignored_attributes: dict = settings.get("mqtt.ignored_attributes") assert (ignored_attributes is None) or ( - len(ignored_attributes) > 0 + isinstance(ignored_attributes, dict) ), f"Configuration 'mqtt.ignored_attributes':{ignored_attributes} is not a valid dict" timestamp_attribute: str = settings.get("mqtt.timestamp_attribute", @@ -159,7 +159,7 @@ def test_connectivity_to_graphdb(): host: str = parsed[0] port: int = None if (len(parsed) == 2): - port: int = parsed[1] + port: int = int(parsed[1]) if (port is None or port == ""): port = 7687 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) diff --git a/03_uns_graphdb/test/test_uns_graphdb.py b/03_uns_graphdb/test/test_uns_graphdb.py index 19fb4309..8df8230f 100644 --- a/03_uns_graphdb/test/test_uns_graphdb.py +++ b/03_uns_graphdb/test/test_uns_graphdb.py @@ -30,6 +30,10 @@ def test_Uns_MQTT_GraphDb(): try: uns_mqtt_graphdb = Uns_MQTT_GraphDb() assert uns_mqtt_graphdb is not None, "Connection to either the MQTT Broker or the Graph DB did not happen" + except Exception as ex: + pytest.fail( + f"Connection to either the MQTT Broker or the Graph DB did not happen: Exception {ex}" + ) finally: if (uns_mqtt_graphdb is not None): uns_mqtt_graphdb.uns_client.disconnect() diff --git a/04_uns_historian/Readme.md b/04_uns_historian/Readme.md index 6c9969cb..87332e87 100644 --- a/04_uns_historian/Readme.md +++ b/04_uns_historian/Readme.md @@ -56,7 +56,7 @@ docker stop uns_timescaledb # ``` ## Key Configurations to provide This application has two configuration file. -1. [settings.yaml](./settings.yaml): Contain the key configurations need to connect with MQTT brokers as well as timescale db +1. [settings.yaml](./conf/settings.yaml): Contain the key configurations need to connect with MQTT brokers as well as timescale db **key** | **sub key** | **description** | ***default value*** | ------ | ------ | ------ | ------ **mqtt** | **host**\*| Hostname of the mqtt broker instant. Mandatory configuration | *None* @@ -66,7 +66,7 @@ This application has two configuration file. mqtt | keep_alive | Maximum time interval in seconds between two control packet published by the client (int) | *60* mqtt | reconnect_on_failure | Makes the client handle reconnection(s). Recommend keeping this True (True,False)| *True* mqtt | version | The MQTT version to be used for connecting to the broker. Valid values are : 5 (for MQTTv5), 4 (for MQTTv311) , 3(for MQTTv31) | *5* - mqtt | transp ort | Valid values are "websockets", "tcp" | *"tcp"* + mqtt | transport | Valid values are "websockets", "tcp" | *"tcp"* mqtt | ignored_attributes | Map of topic & list of attributes which are to be ignored from persistance. supports wild cards for topics and nested via . notation for the attributes
e.g.
{
'topic1' : ["attr1", "attr2", "attr2.subAttr1" ],
'topic2/+' : ["A", "A.B.C"],
'topic3/#' : ["x", "Y"]
} | None mqtt | timestamp_attribute | the attribute name which should contain the timestamp of the message's publishing| *"timestamp"* **historian** | **hostname**\* | Mandatory. The db hostname of your TimescaleDB instance| *None* @@ -75,7 +75,7 @@ This application has two configuration file. **historian** | **table**\*| Mandatory. The hypertable where the time-series of messages is stored. See [db script](./sql_scripts/02_setup_hypertable.sql)| *None* **dynaconf_merge**\* | | Mandatory param. Always keep value as true | -1. [.secret.yaml](./.secrets_template.yaml) : Contains the username and passwords to connect to the MQTT cluster and the timescaledb +1. [.secret.yaml](./conf/.secrets_template.yaml) : Contains the username and passwords to connect to the MQTT cluster and the timescaledb This file is not checked into the repository for security purposes. However there is a template file provided **`.secrets_template.yaml`** which should be edited and renamed to **`.secrets.yaml`** **key** | **sub key** | **sub key** | **description** | ***default value*** | :------ | :------ | :------ | :------ | :------ @@ -94,7 +94,7 @@ This application has two configuration file. historian | sslmode | | Enables encrypted connection to TimescaleDB. valid values are disable, allow, prefer, require, verify-ca, verify-full | *None* **dynaconf_merge**\* | | | Mandatory param. Always keep value as true | ## Running the python script -This function is executed by the following command with the current folder as `03_uns_graphdb` +This function is executed by the following command with the current folder as `04_uns_historian` ```bash # install virtual env python -m pip install --user virtualenv @@ -110,8 +110,10 @@ The set of test for this module is executed by ```python source env_historian/bin/activate python -m pip install -r requirements_dev.txt +#run all tests excluding integration tests +pytest -m "not integrationtest" test/ +# runs all tests pytest test/ -TBD ``` diff --git a/04_uns_historian/conf/settings.yaml b/04_uns_historian/conf/settings.yaml index 622b3a81..74f9576c 100644 --- a/04_uns_historian/conf/settings.yaml +++ b/04_uns_historian/conf/settings.yaml @@ -8,7 +8,7 @@ mqtt: version : 5 # Default is MQTTv5. Valid values are : 5 (for MQTTv5), 4 (for MQTTv311) , 3(for MQTTv31) #clean_session: false # specify this only if the protocol is not MQTTv5 transport : "tcp" # Default is tcp. Valid values are: websockets, tcp - ignored_attributes : {'+': "timestamp"} # dict of topic , list of attributes which are to be ignored from persistance. + ignored_attributes : { } # dict of topic , list of attributes which are to be ignored from persistance. # e.g. {'topic1' : ["attr1", "attr2", "attr2.subAttr1" ], # 'topic2/+' : ["A", "A.B.C"], # 'topic3/#' : ["x", "Y"]} diff --git a/04_uns_historian/requirements.txt b/04_uns_historian/requirements.txt index 71a590d6..cec2f3b4 100644 --- a/04_uns_historian/requirements.txt +++ b/04_uns_historian/requirements.txt @@ -1,6 +1,6 @@ -wheel +wheel>=0.37.1 psycopg2-binary>=2.9.3 -pytz>=2021.3 +pytz>=2022.1 paho-mqtt>=1.6.1 -logger -dynaconf \ No newline at end of file +logger>=1.4 +dynaconf>=3.1.9 \ No newline at end of file diff --git a/04_uns_historian/setup.cfg b/04_uns_historian/setup.cfg index e69e5b8e..b6a0477e 100644 --- a/04_uns_historian/setup.cfg +++ b/04_uns_historian/setup.cfg @@ -15,7 +15,7 @@ classifiers = package_dir = = src packages = find: -python_requires = >=3.8 +python_requires = >=3.10.5 [options.packages.find] where = src \ No newline at end of file diff --git a/04_uns_historian/src/uns_historian/historian_handler.py b/04_uns_historian/src/uns_historian/historian_handler.py index 6db6e99a..05360d5b 100644 --- a/04_uns_historian/src/uns_historian/historian_handler.py +++ b/04_uns_historian/src/uns_historian/historian_handler.py @@ -65,7 +65,7 @@ def connect(self, retry: int = 0): raise ex else: retry += 1 - LOGGER.error("Error Connecting to %s. Error:", + LOGGER.error("Error Connecting to %s. Error: %s", self.database, str(ex), stack_info=True, @@ -73,11 +73,12 @@ def connect(self, retry: int = 0): time.sleep(SLEEP_BTW_ATTEMPT) self.connect(retry=retry) except Exception as ex: - LOGGER.error("Error Connecting to %s. Unable to retry. Error:", - self.database, - str(ex), - stack_info=True, - exc_info=True) + LOGGER.error( + "Error Connecting to %s. Unable to retry. Error:%s", + self.database, + str(ex), + stack_info=True, + exc_info=True) raise ex def getcursor(self): diff --git a/04_uns_historian/test/test_historian_config.py b/04_uns_historian/test/test_historian_config.py index 232a222c..c81e237a 100644 --- a/04_uns_historian/test/test_historian_config.py +++ b/04_uns_historian/test/test_historian_config.py @@ -87,7 +87,7 @@ def test_mqtt_config(): ignored_attributes: dict = settings.get("mqtt.ignored_attributes") assert (ignored_attributes is None) or ( - len(ignored_attributes) > 0 + isinstance(ignored_attributes, dict) ), f"Configuration 'mqtt.ignored_attributes':{ignored_attributes} is not a valid dict" timestamp_attribute: str = settings.get("mqtt.timestamp_attribute", diff --git a/04_uns_historian/test/test_uns_historian.py b/04_uns_historian/test/test_uns_historian.py index 3f239fa7..091201ee 100644 --- a/04_uns_historian/test/test_uns_historian.py +++ b/04_uns_historian/test/test_uns_historian.py @@ -30,6 +30,10 @@ def test_Uns_Mqtt_Historian(): try: uns_mqtt_historian = Uns_Mqtt_Historian() assert uns_mqtt_historian is not None, "Connection to either the MQTT Broker or the Historian DB did not happen" + except Exception as ex: + pytest.fail( + f"Connection to either the MQTT Broker or the Historian DB did not happen: Exception {ex}" + ) finally: if (uns_mqtt_historian is not None): uns_mqtt_historian.uns_client.disconnect() diff --git a/05_sparkplugb/.gitignore b/05_sparkplugb/.gitignore new file mode 100644 index 00000000..9ef8d557 --- /dev/null +++ b/05_sparkplugb/.gitignore @@ -0,0 +1,2 @@ +__pycache__ +env_* diff --git a/05_sparkplugb/.vscode/launch.json b/05_sparkplugb/.vscode/launch.json new file mode 100644 index 00000000..2b2502c6 --- /dev/null +++ b/05_sparkplugb/.vscode/launch.json @@ -0,0 +1,16 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Python: Current File", + "type": "python", + "request": "launch", + "program": "${file}", + "console": "integratedTerminal", + "justMyCode": false + } + ] +} \ No newline at end of file diff --git a/05_sparkplugb/.vscode/settings.json b/05_sparkplugb/.vscode/settings.json new file mode 100644 index 00000000..7eec4547 --- /dev/null +++ b/05_sparkplugb/.vscode/settings.json @@ -0,0 +1,37 @@ +{ + "cSpell.words": [ + "DBIRTH", + "DCMD", + "DDATA", + "DDEATH", + "IIOT", + "NBIRTH", + "NCMD", + "NDATA", + "NDEATH", + "integrationtest", + "k8scluster", + "Kubernetes", + "microk8s", + "mqtt", + "paho", + "proto", + "protobuf", + "pytest", + "sparkplugb", + "sslmode", + "venv", + "virtualenv" + ], + "python.analysis.extraPaths": [ + "src", + "../02_mqtt-cluster/src", + "./src/uns_sparkplugb" + ], + "python.testing.pytestArgs": [ + "test" + ], + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true, + "python.formatting.provider": "yapf", +} \ No newline at end of file diff --git a/05_sparkplugb/Readme.md b/05_sparkplugb/Readme.md new file mode 100644 index 00000000..e01a556e --- /dev/null +++ b/05_sparkplugb/Readme.md @@ -0,0 +1,187 @@ +[![SparkplugB Decoder](https://github.com/mkashwin/unifiednamespace/actions/workflows/uns_sparkplugb-app.yml/badge.svg)](https://github.com/mkashwin/unifiednamespace/actions/workflows/uns_sparkplugb-app.yml) + +# Decoder Application to process SparkPlugB +This SparkplugB decoder is an MQTT Application Node deployed on the edge with the following functionality +- subscribes to the Topic *`spBv1.0/#`* +- decodes the payload received and extract the metrics received +- enhance the decoded message to add + - *\ + - *\ + - *\ *if provided* +- extract metric name to get ISA-95 UNS namespace +- if metric alias is used, map the metric alias to previously provided metric name +- collate all metrics specific to one metric name and create a consolidated message payload +- publish consolidate message payload to the UNS namespace + +This is **not** a SCADA/IIOT host and will not be publishing any control messages to the broker + +# SparkplugB™ Topic Namespace +SparkplugB Topic namespace follows the following structure +>**spBv1.0**/*\*/*\*/*\*/*[\]* + +Where in: +* *\*: provides for a logical grouping of MQTT EoN nodes +* *\* provides an indication as to how to handle the MQTT payload of the message. The following message_type elements are defined for the SparkplugB™ Topic Namespace: + - NBIRTH: Birth certificate for MQTT EoN nodes. + - NDEATH: Death certificate for MQTT EoN nodes. + - DBIRTH: Birth certificate for Devices. + - DDEATH: Death certificate for Devices. + - NDATA: Node data message. + - DDATA: Device data message. + - NCMD: Node command message. + - DCMD: Device command message. + - STATE: Critical application state message + Please refer to the detailed specification of these message types in the [SparkplugB Specs](https://www.eclipse.org/tahu/spec/Sparkplug%20Topic%20Namespace%20and%20State%20ManagementV2.2-with%20appendix%20B%20format%20-%20Eclipse.pdf) +* *\*: uniquely identifies the MQTT EoN node within the factory context. +* *[\]*: optional and identifies a device attached (physically or logically) to the MQTT EoN node + +## Mapping logic for SparkplugB to ISA-95 +With reference to the article [Using Sparkplug to Map ISA 95](https://www.hivemq.com/solutions/manufacturing/smart-manufacturing-using-isa95-mqtt-sparkplug-and-uns/), each SparkPlugB message contains an attribute **name** which contains the ISA namespace for the message in addition to the tag. +e.g. +```json +{ +"timestamp": 1486144502122, +"metrics": [{ +"name": "Enterprise/Site/Area/Line/Cell/Tag1", +"alias": 1, +"timestamp": 1479123452194, +"dataType": "String", +"isHistorical": false, +"value": "Test" +}, +{ +"name": "Enterprise/Site/Area/Line/Tag2", +"alias": 2, +"timestamp": 1479123452196, +"dataType": "String", +"isHistorical": false, +"value": "LineTest" +} + +], +"seq": 2 +} +``` +When we parse the metric name we get + - The topic name in the UNS i.e. `Enterprise/Site/Area/Line/Cell` + - The name of the tag i.e. `Tag1` and `Tag2` which will now be added as an attribute to the JSON object being published to the UNS + +The spB payload may contain multiple metrics which belong to different topics hence we merge all tags belonging to the name UNS topic. The logic for merging is as following + - the name of the metric will be split into the topic and tag + - tags for the same topic are merged in to one JSON object + - if a metric is historical, we create an array of tag values with the latest version being at 0 position + - if a metric name is duplicated and not historical, the newer tag value will override the older tag value in the JSON object + - the overall timestamp for the JSON object is the max of timestamps of all the tags + + + +This mapping is to be done only for the message types + - NDATA + - DDATA + - NCMD + - DCMD +***IMPORTANT: The birth and death messages are not mapped to the UNS currently*** + + + +# Preparation steps required to setup protocol buffer and SparkplugB dependencies +1. **Step 1**: Download or install protoc. Refer + - [Installing on Linux/MacOs](https://grpc.io/docs/protoc-installation/) + - [Install pre-compiled version](https://github.com/protocolbuffers/protobuf/releases). This project currently is using version [Protocol Buffers v3.19.4](https://github.com/protocolbuffers/protobuf/releases/tag/v3.19.4) + and downloaded the pre-compiled versions for linux-x86_64 and win64. For other platforms please replace with the appropriate runtime or compile the runtime directly +1. **Step 2**: Copy the [SparkPlugB protocol buffer specification](https://github.com/eclipse/tahu/tree/master/sparkplug_b/sparkplug_b.proto) from [Eclipse Tahu project](https://github.com/eclipse/tahu/tree/master/sparkplug_b) to the folder [./sparkplug_b](./sparkplug_b/) +1. **Step 3**: Compile the SparkplugB protocol buffer into python class by the following command + > + ```bash + # Execute on Linux + ./protobuf/bin/protoc -I ./sparkplug_b/ --python_out=./src/uns_sparkplugb/generated ./sparkplug_b/sparkplug_b.proto + ``` + > + ```powershell + # Execute on windows + .\protobuf\bin\protoc.exe -I .\sparkplug_b\ --python_out=.\src\uns_sparkplugb\generated .\sparkplug_b\sparkplug_b.proto + ``` +## Key Configurations to provide +This application has two configuration file. +1. [settings.yaml](./conf/settings.yaml): Contain the key configurations need to connect with MQTT brokers as well as timescale db + **key** | **sub key** | **description** | ***default value*** | + ------ | ------ | ------ | ------ + **mqtt** | **host**\*| Hostname of the mqtt broker instant. Mandatory configuration | *None* + mqtt | port | Port of the mqtt broker (int) | *1883* + mqtt | topic | Must be in the names space of SpB i.e. **spBv1.0/#** | *spBv1.0/#* + mqtt | qos | QOS for the subscription. Valid values are 0,1,2 | *1* + mqtt | keep_alive | Maximum time interval in seconds between two control packet published by the client (int) | *60* + mqtt | reconnect_on_failure | Makes the client handle reconnection(s). Recommend keeping this True (True,False)| *True* + mqtt | version | The MQTT version to be used for connecting to the broker. Valid values are : 5 (for MQTTv5), 4 (for MQTTv311) , 3(for MQTTv31) | *5* + mqtt | transport | Valid values are "websockets", "tcp" | *"tcp"* + mqtt | ignored_attributes | Map of topic & list of attributes which are to be ignored from persistance. supports wild cards for topics and nested via . notation for the attributes
e.g.
{
'topic1' : ["attr1", "attr2", "attr2.subAttr1" ],
'topic2/+' : ["A", "A.B.C"],
'topic3/#' : ["x", "Y"]
} | None + mqtt | timestamp_attribute | the attribute name which should contain the timestamp of the message's publishing| *"timestamp"* + sparkplugb | | *currently empty*| + **dynaconf_merge**\* | | Mandatory param. Always keep value as true | + +1. [.secret.yaml](./conf/.secrets_template.yaml) : Contains the username and passwords to connect to the MQTT cluster and the timescaledb + This file is not checked into the repository for security purposes. However there is a template file provided [**`.secrets_template.yaml`**](./conf/.secrets_template.yaml) which should be edited and renamed to **`.secrets.yaml`** + **key** | **sub key** | **sub key** | **description** | ***default value*** | + :------ | :------ | :------ | :------ | :------ + mqtt | username | | The user id needed to authenticate with the MQTT broker | *None* + mqtt | password | | The password needed to authenticate with the MQTT broker | *None* + mqtt | tls | |Provide a map of attributes needed for a TLS connection to the MQTT broker. See below attributes | *None* + mqtt | tls | ca_certs | fully qualified path to the ca cert file. Mandatory for an SSL connection | *None* + mqtt | tls | certfile | fully qualified path to the cert file | *None* + mqtt | tls | keyfile | fully qualified path to the keyfile for the cert| *None* + mqtt | tls | cert_reqs | Boolean. If note provided then ssl.CERT_NONE is used. if True the ssl.CERT_REQUIRED is used. else ssl.CERT_OPTIONAL is used | *None* + mqtt | tls | ciphers | Specify which encryption ciphers are allowed for this connection | *None* + mqtt | tls | keyfile_password | Password used to encrypt your certfile and keyfile | *None* + mqtt | tls | insecure_cert | Boolean. Skips hostname checking required for self signed certificates. | *True* + **dynaconf_merge**\* | | | Mandatory param. Always keep value as true | + + +## Running the python script +This function is executed by the following command with the current folder as [`05_sparkplugb`](.) +```bash +# install virtual env +python -m pip install --user virtualenv +python -m venv env_sparkplugb +source env_sparkplugb/bin/activate +python -m pip install --upgrade pip +python -m pip install --upgrade -r requirements.txt +python ./src/uns_sparkplugb/uns_sparkplugb.py +``` + +### Running tests +The set of test for this module is executed by +```bash +source env_sparkplugb/bin/activate +python -m pip install -r requirements_dev.txt +#run all tests excluding integration tests +pytest -m "not integrationtest" test/ +# runs all tests +pytest test/ +``` +# Reference +* [Eclipse Sparkplug B Specification](https://www.eclipse.org/tahu/spec/Sparkplug%20Topic%20Namespace%20and%20State%20ManagementV2.2-with%20appendix%20B%20format%20-%20Eclipse.pdf) +* [Cirrus Link Sparkplug B MQTT Tutorials](https://docs.chariot.io/display/CLD79/B%3A+Example+Python+Client) +* [Github Eclipse Tahu project](https://github.com/eclipse/tahu) +* [Google Protocol Buffers Project](https://github.com/protocolbuffers/protobuf) + + +## Limitations +1. The proto files were not being compiled correctly with [Protobuf Ver 3.20.0 and higher](https://github.com/protocolbuffers/protobuf/releases/tag/v3.20.0) hence I had to downgrade the protobuf version to [Protobuf v3.19.4](https://github.com/protocolbuffers/protobuf/releases/tag/v3.19.4) + +1. The protoc executable for [Linux](./protobuf/bin/protoc) is for x86_64 architecture and will need execute rights to be able to run and compile the [sparkplug_b.proto](./sparkplug_b/sparkplug_b.proto) specification. For other architectures please download the appropriate pre compiled version of [Protobuf release v3.19.4](https://github.com/protocolbuffers/protobuf/releases/tag/v3.19.4) e.g. + - [protoc-3.19.4-linux-aarch_64.zip](https://github.com/protocolbuffers/protobuf/releases/download/v3.19.4/protoc-3.19.4-linux-aarch_64.zip) + - [protoc-3.19.4-linux-ppcle_64.zip](https://github.com/protocolbuffers/protobuf/releases/download/v3.19.4/protoc-3.19.4-linux-ppcle_64.zip) + - [protoc-3.19.4-linux-s390_64.zip](https://github.com/protocolbuffers/protobuf/releases/download/v3.19.4/protoc-3.19.4-linux-s390_64.zip) + - [protoc-3.19.4-linux-x86_32.zip](https://github.com/protocolbuffers/protobuf/releases/download/v3.19.4/protoc-3.19.4-linux-x86_32.zip) + +1. The application assumes the the MQTT broker for SparkPlugB and the UNS are one and the same as it does not sense to have separate brokers for the same. This can be enhanced easily if there is a requirement for the same. Please create issue on the Github project +1. Need to understand how to handle NBIRTH, NDEATH, DBIRTH & DDEATH message types +1. Need to understand how to handle metric types PropertySet, PropertySetList, Array +1. Need to understand how to handle metric fields : metadata, properties, is_multi_part \ No newline at end of file diff --git a/05_sparkplugb/conf/.gitignore b/05_sparkplugb/conf/.gitignore new file mode 100644 index 00000000..20b14e65 --- /dev/null +++ b/05_sparkplugb/conf/.gitignore @@ -0,0 +1,3 @@ +# Ignore dynaconf secret files +.secrets.* +env_* diff --git a/05_sparkplugb/conf/.secrets_template.yaml b/05_sparkplugb/conf/.secrets_template.yaml new file mode 100644 index 00000000..ac466cce --- /dev/null +++ b/05_sparkplugb/conf/.secrets_template.yaml @@ -0,0 +1,14 @@ +mqtt: + username: '#' + password: '#' + tls: # keep all properties commented if SSL configuration is not needed + #ca_certs: # YOUR/ROOT/CA/PATH" Mandatory for an SSL connection + #certfile: # YOUR/DEVICE/CERT/PATH . Optional + #keyfile: # YOUR/DEVICE/KEY/PATH. Optional + #cert_reqs: # Boolean. If None then ssl.CERT_NONE is used. if True the ssl.CERT_REQUIRED is used. + # else ssl.CERT_OPTIONAL is used .Optional + #ciphers: # Str. specifying which encryption ciphers are allowed for this connection. Optional + #keyfile_password: # Str. password used to encrypt your certfile and keyfile. Optional + #insecure_cert: True # Boolean. Skips hostname checking required for self signed certificates. Optional + +dynaconf_merge: true \ No newline at end of file diff --git a/05_sparkplugb/conf/settings.yaml b/05_sparkplugb/conf/settings.yaml new file mode 100644 index 00000000..365bc98f --- /dev/null +++ b/05_sparkplugb/conf/settings.yaml @@ -0,0 +1,21 @@ +mqtt: + host: "broker.emqx.io" # must be specified. No defaults e.g.198.168.220.3 broker.emqx.io + port: 1883 # Default value 1883 if not specified + topic: "spBv1.0/#" # Do not change from spBv1.0/# + qos: 1 # Default value is 1. Recommend 1 or 2. Do not use 0 + keep_alive: 60 + reconnect_on_failure: True + version: 5 # Default is MQTTv5. Valid values are : 5 (for MQTTv5), 4 (for MQTTv311) , 3(for MQTTv31) + #clean_session: false # specify this only if the protocol is not MQTTv5 + transport: "tcp" # Default is tcp. Valid values are: websockets, tcp + ignored_attributes: + {} # dict of topic , list of attributes which are to be ignored from persistance. + # e.g. {'topic1' : ["attr1", "attr2", "attr2.subAttr1" ], + # 'topic2/+' : ["A", "A.B.C"], + # 'topic3/#' : ["x", "Y"]} + timestamp_attribute: "timestamp" # the attribute name which should contain the timestamp of the message's publishing + +sparkplugb: + # SPB namespace is spBv1.0////<[device_id]> + +dynaconf_merge: true diff --git a/05_sparkplugb/protobuf/bin/protoc b/05_sparkplugb/protobuf/bin/protoc new file mode 100644 index 00000000..16eb6c53 Binary files /dev/null and b/05_sparkplugb/protobuf/bin/protoc differ diff --git a/05_sparkplugb/protobuf/bin/protoc.exe b/05_sparkplugb/protobuf/bin/protoc.exe new file mode 100644 index 00000000..c3925908 Binary files /dev/null and b/05_sparkplugb/protobuf/bin/protoc.exe differ diff --git a/05_sparkplugb/protobuf/include/google/protobuf/any.proto b/05_sparkplugb/protobuf/include/google/protobuf/any.proto new file mode 100644 index 00000000..6ed8a23c --- /dev/null +++ b/05_sparkplugb/protobuf/include/google/protobuf/any.proto @@ -0,0 +1,158 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option go_package = "google.golang.org/protobuf/types/known/anypb"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "AnyProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; + +// `Any` contains an arbitrary serialized protocol buffer message along with a +// URL that describes the type of the serialized message. +// +// Protobuf library provides support to pack/unpack Any values in the form +// of utility functions or additional generated methods of the Any type. +// +// Example 1: Pack and unpack a message in C++. +// +// Foo foo = ...; +// Any any; +// any.PackFrom(foo); +// ... +// if (any.UnpackTo(&foo)) { +// ... +// } +// +// Example 2: Pack and unpack a message in Java. +// +// Foo foo = ...; +// Any any = Any.pack(foo); +// ... +// if (any.is(Foo.class)) { +// foo = any.unpack(Foo.class); +// } +// +// Example 3: Pack and unpack a message in Python. +// +// foo = Foo(...) +// any = Any() +// any.Pack(foo) +// ... +// if any.Is(Foo.DESCRIPTOR): +// any.Unpack(foo) +// ... +// +// Example 4: Pack and unpack a message in Go +// +// foo := &pb.Foo{...} +// any, err := anypb.New(foo) +// if err != nil { +// ... +// } +// ... +// foo := &pb.Foo{} +// if err := any.UnmarshalTo(foo); err != nil { +// ... +// } +// +// The pack methods provided by protobuf library will by default use +// 'type.googleapis.com/full.type.name' as the type URL and the unpack +// methods only use the fully qualified type name after the last '/' +// in the type URL, for example "foo.bar.com/x/y.z" will yield type +// name "y.z". +// +// +// JSON +// ==== +// The JSON representation of an `Any` value uses the regular +// representation of the deserialized, embedded message, with an +// additional field `@type` which contains the type URL. Example: +// +// package google.profile; +// message Person { +// string first_name = 1; +// string last_name = 2; +// } +// +// { +// "@type": "type.googleapis.com/google.profile.Person", +// "firstName": , +// "lastName": +// } +// +// If the embedded message type is well-known and has a custom JSON +// representation, that representation will be embedded adding a field +// `value` which holds the custom JSON in addition to the `@type` +// field. Example (for message [google.protobuf.Duration][]): +// +// { +// "@type": "type.googleapis.com/google.protobuf.Duration", +// "value": "1.212s" +// } +// +message Any { + // A URL/resource name that uniquely identifies the type of the serialized + // protocol buffer message. This string must contain at least + // one "/" character. The last segment of the URL's path must represent + // the fully qualified name of the type (as in + // `path/google.protobuf.Duration`). The name should be in a canonical form + // (e.g., leading "." is not accepted). + // + // In practice, teams usually precompile into the binary all types that they + // expect it to use in the context of Any. However, for URLs which use the + // scheme `http`, `https`, or no scheme, one can optionally set up a type + // server that maps type URLs to message definitions as follows: + // + // * If no scheme is provided, `https` is assumed. + // * An HTTP GET on the URL must yield a [google.protobuf.Type][] + // value in binary format, or produce an error. + // * Applications are allowed to cache lookup results based on the + // URL, or have them precompiled into a binary to avoid any + // lookup. Therefore, binary compatibility needs to be preserved + // on changes to types. (Use versioned type names to manage + // breaking changes.) + // + // Note: this functionality is not currently available in the official + // protobuf release, and it is not used for type URLs beginning with + // type.googleapis.com. + // + // Schemes other than `http`, `https` (or the empty scheme) might be + // used with implementation specific semantics. + // + string type_url = 1; + + // Must be a valid serialized protocol buffer of the above specified type. + bytes value = 2; +} diff --git a/05_sparkplugb/protobuf/include/google/protobuf/api.proto b/05_sparkplugb/protobuf/include/google/protobuf/api.proto new file mode 100644 index 00000000..3d598fc8 --- /dev/null +++ b/05_sparkplugb/protobuf/include/google/protobuf/api.proto @@ -0,0 +1,208 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +import "google/protobuf/source_context.proto"; +import "google/protobuf/type.proto"; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "ApiProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; +option go_package = "google.golang.org/protobuf/types/known/apipb"; + +// Api is a light-weight descriptor for an API Interface. +// +// Interfaces are also described as "protocol buffer services" in some contexts, +// such as by the "service" keyword in a .proto file, but they are different +// from API Services, which represent a concrete implementation of an interface +// as opposed to simply a description of methods and bindings. They are also +// sometimes simply referred to as "APIs" in other contexts, such as the name of +// this message itself. See https://cloud.google.com/apis/design/glossary for +// detailed terminology. +message Api { + // The fully qualified name of this interface, including package name + // followed by the interface's simple name. + string name = 1; + + // The methods of this interface, in unspecified order. + repeated Method methods = 2; + + // Any metadata attached to the interface. + repeated Option options = 3; + + // A version string for this interface. If specified, must have the form + // `major-version.minor-version`, as in `1.10`. If the minor version is + // omitted, it defaults to zero. If the entire version field is empty, the + // major version is derived from the package name, as outlined below. If the + // field is not empty, the version in the package name will be verified to be + // consistent with what is provided here. + // + // The versioning schema uses [semantic + // versioning](http://semver.org) where the major version number + // indicates a breaking change and the minor version an additive, + // non-breaking change. Both version numbers are signals to users + // what to expect from different versions, and should be carefully + // chosen based on the product plan. + // + // The major version is also reflected in the package name of the + // interface, which must end in `v`, as in + // `google.feature.v1`. For major versions 0 and 1, the suffix can + // be omitted. Zero major versions must only be used for + // experimental, non-GA interfaces. + // + // + string version = 4; + + // Source context for the protocol buffer service represented by this + // message. + SourceContext source_context = 5; + + // Included interfaces. See [Mixin][]. + repeated Mixin mixins = 6; + + // The source syntax of the service. + Syntax syntax = 7; +} + +// Method represents a method of an API interface. +message Method { + // The simple name of this method. + string name = 1; + + // A URL of the input message type. + string request_type_url = 2; + + // If true, the request is streamed. + bool request_streaming = 3; + + // The URL of the output message type. + string response_type_url = 4; + + // If true, the response is streamed. + bool response_streaming = 5; + + // Any metadata attached to the method. + repeated Option options = 6; + + // The source syntax of this method. + Syntax syntax = 7; +} + +// Declares an API Interface to be included in this interface. The including +// interface must redeclare all the methods from the included interface, but +// documentation and options are inherited as follows: +// +// - If after comment and whitespace stripping, the documentation +// string of the redeclared method is empty, it will be inherited +// from the original method. +// +// - Each annotation belonging to the service config (http, +// visibility) which is not set in the redeclared method will be +// inherited. +// +// - If an http annotation is inherited, the path pattern will be +// modified as follows. Any version prefix will be replaced by the +// version of the including interface plus the [root][] path if +// specified. +// +// Example of a simple mixin: +// +// package google.acl.v1; +// service AccessControl { +// // Get the underlying ACL object. +// rpc GetAcl(GetAclRequest) returns (Acl) { +// option (google.api.http).get = "/v1/{resource=**}:getAcl"; +// } +// } +// +// package google.storage.v2; +// service Storage { +// rpc GetAcl(GetAclRequest) returns (Acl); +// +// // Get a data record. +// rpc GetData(GetDataRequest) returns (Data) { +// option (google.api.http).get = "/v2/{resource=**}"; +// } +// } +// +// Example of a mixin configuration: +// +// apis: +// - name: google.storage.v2.Storage +// mixins: +// - name: google.acl.v1.AccessControl +// +// The mixin construct implies that all methods in `AccessControl` are +// also declared with same name and request/response types in +// `Storage`. A documentation generator or annotation processor will +// see the effective `Storage.GetAcl` method after inheriting +// documentation and annotations as follows: +// +// service Storage { +// // Get the underlying ACL object. +// rpc GetAcl(GetAclRequest) returns (Acl) { +// option (google.api.http).get = "/v2/{resource=**}:getAcl"; +// } +// ... +// } +// +// Note how the version in the path pattern changed from `v1` to `v2`. +// +// If the `root` field in the mixin is specified, it should be a +// relative path under which inherited HTTP paths are placed. Example: +// +// apis: +// - name: google.storage.v2.Storage +// mixins: +// - name: google.acl.v1.AccessControl +// root: acls +// +// This implies the following inherited HTTP annotation: +// +// service Storage { +// // Get the underlying ACL object. +// rpc GetAcl(GetAclRequest) returns (Acl) { +// option (google.api.http).get = "/v2/acls/{resource=**}:getAcl"; +// } +// ... +// } +message Mixin { + // The fully qualified name of the interface which is included. + string name = 1; + + // If non-empty specifies a path under which inherited HTTP paths + // are rooted. + string root = 2; +} diff --git a/05_sparkplugb/protobuf/include/google/protobuf/compiler/plugin.proto b/05_sparkplugb/protobuf/include/google/protobuf/compiler/plugin.proto new file mode 100644 index 00000000..9242aacc --- /dev/null +++ b/05_sparkplugb/protobuf/include/google/protobuf/compiler/plugin.proto @@ -0,0 +1,183 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Author: kenton@google.com (Kenton Varda) +// +// WARNING: The plugin interface is currently EXPERIMENTAL and is subject to +// change. +// +// protoc (aka the Protocol Compiler) can be extended via plugins. A plugin is +// just a program that reads a CodeGeneratorRequest from stdin and writes a +// CodeGeneratorResponse to stdout. +// +// Plugins written using C++ can use google/protobuf/compiler/plugin.h instead +// of dealing with the raw protocol defined here. +// +// A plugin executable needs only to be placed somewhere in the path. The +// plugin should be named "protoc-gen-$NAME", and will then be used when the +// flag "--${NAME}_out" is passed to protoc. + +syntax = "proto2"; + +package google.protobuf.compiler; +option java_package = "com.google.protobuf.compiler"; +option java_outer_classname = "PluginProtos"; + +option go_package = "google.golang.org/protobuf/types/pluginpb"; + +import "google/protobuf/descriptor.proto"; + +// The version number of protocol compiler. +message Version { + optional int32 major = 1; + optional int32 minor = 2; + optional int32 patch = 3; + // A suffix for alpha, beta or rc release, e.g., "alpha-1", "rc2". It should + // be empty for mainline stable releases. + optional string suffix = 4; +} + +// An encoded CodeGeneratorRequest is written to the plugin's stdin. +message CodeGeneratorRequest { + // The .proto files that were explicitly listed on the command-line. The + // code generator should generate code only for these files. Each file's + // descriptor will be included in proto_file, below. + repeated string file_to_generate = 1; + + // The generator parameter passed on the command-line. + optional string parameter = 2; + + // FileDescriptorProtos for all files in files_to_generate and everything + // they import. The files will appear in topological order, so each file + // appears before any file that imports it. + // + // protoc guarantees that all proto_files will be written after + // the fields above, even though this is not technically guaranteed by the + // protobuf wire format. This theoretically could allow a plugin to stream + // in the FileDescriptorProtos and handle them one by one rather than read + // the entire set into memory at once. However, as of this writing, this + // is not similarly optimized on protoc's end -- it will store all fields in + // memory at once before sending them to the plugin. + // + // Type names of fields and extensions in the FileDescriptorProto are always + // fully qualified. + repeated FileDescriptorProto proto_file = 15; + + // The version number of protocol compiler. + optional Version compiler_version = 3; + +} + +// The plugin writes an encoded CodeGeneratorResponse to stdout. +message CodeGeneratorResponse { + // Error message. If non-empty, code generation failed. The plugin process + // should exit with status code zero even if it reports an error in this way. + // + // This should be used to indicate errors in .proto files which prevent the + // code generator from generating correct code. Errors which indicate a + // problem in protoc itself -- such as the input CodeGeneratorRequest being + // unparseable -- should be reported by writing a message to stderr and + // exiting with a non-zero status code. + optional string error = 1; + + // A bitmask of supported features that the code generator supports. + // This is a bitwise "or" of values from the Feature enum. + optional uint64 supported_features = 2; + + // Sync with code_generator.h. + enum Feature { + FEATURE_NONE = 0; + FEATURE_PROTO3_OPTIONAL = 1; + } + + // Represents a single generated file. + message File { + // The file name, relative to the output directory. The name must not + // contain "." or ".." components and must be relative, not be absolute (so, + // the file cannot lie outside the output directory). "/" must be used as + // the path separator, not "\". + // + // If the name is omitted, the content will be appended to the previous + // file. This allows the generator to break large files into small chunks, + // and allows the generated text to be streamed back to protoc so that large + // files need not reside completely in memory at one time. Note that as of + // this writing protoc does not optimize for this -- it will read the entire + // CodeGeneratorResponse before writing files to disk. + optional string name = 1; + + // If non-empty, indicates that the named file should already exist, and the + // content here is to be inserted into that file at a defined insertion + // point. This feature allows a code generator to extend the output + // produced by another code generator. The original generator may provide + // insertion points by placing special annotations in the file that look + // like: + // @@protoc_insertion_point(NAME) + // The annotation can have arbitrary text before and after it on the line, + // which allows it to be placed in a comment. NAME should be replaced with + // an identifier naming the point -- this is what other generators will use + // as the insertion_point. Code inserted at this point will be placed + // immediately above the line containing the insertion point (thus multiple + // insertions to the same point will come out in the order they were added). + // The double-@ is intended to make it unlikely that the generated code + // could contain things that look like insertion points by accident. + // + // For example, the C++ code generator places the following line in the + // .pb.h files that it generates: + // // @@protoc_insertion_point(namespace_scope) + // This line appears within the scope of the file's package namespace, but + // outside of any particular class. Another plugin can then specify the + // insertion_point "namespace_scope" to generate additional classes or + // other declarations that should be placed in this scope. + // + // Note that if the line containing the insertion point begins with + // whitespace, the same whitespace will be added to every line of the + // inserted text. This is useful for languages like Python, where + // indentation matters. In these languages, the insertion point comment + // should be indented the same amount as any inserted code will need to be + // in order to work correctly in that context. + // + // The code generator that generates the initial file and the one which + // inserts into it must both run as part of a single invocation of protoc. + // Code generators are executed in the order in which they appear on the + // command line. + // + // If |insertion_point| is present, |name| must also be present. + optional string insertion_point = 2; + + // The file contents. + optional string content = 15; + + // Information describing the file content being inserted. If an insertion + // point is used, this information will be appropriately offset and inserted + // into the code generation metadata for the generated files. + optional GeneratedCodeInfo generated_code_info = 16; + } + repeated File file = 15; +} diff --git a/05_sparkplugb/protobuf/include/google/protobuf/descriptor.proto b/05_sparkplugb/protobuf/include/google/protobuf/descriptor.proto new file mode 100644 index 00000000..156e410a --- /dev/null +++ b/05_sparkplugb/protobuf/include/google/protobuf/descriptor.proto @@ -0,0 +1,911 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Author: kenton@google.com (Kenton Varda) +// Based on original Protocol Buffers design by +// Sanjay Ghemawat, Jeff Dean, and others. +// +// The messages in this file describe the definitions found in .proto files. +// A valid .proto file can be translated directly to a FileDescriptorProto +// without any other information (e.g. without reading its imports). + + +syntax = "proto2"; + +package google.protobuf; + +option go_package = "google.golang.org/protobuf/types/descriptorpb"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "DescriptorProtos"; +option csharp_namespace = "Google.Protobuf.Reflection"; +option objc_class_prefix = "GPB"; +option cc_enable_arenas = true; + +// descriptor.proto must be optimized for speed because reflection-based +// algorithms don't work during bootstrapping. +option optimize_for = SPEED; + +// The protocol compiler can output a FileDescriptorSet containing the .proto +// files it parses. +message FileDescriptorSet { + repeated FileDescriptorProto file = 1; +} + +// Describes a complete .proto file. +message FileDescriptorProto { + optional string name = 1; // file name, relative to root of source tree + optional string package = 2; // e.g. "foo", "foo.bar", etc. + + // Names of files imported by this file. + repeated string dependency = 3; + // Indexes of the public imported files in the dependency list above. + repeated int32 public_dependency = 10; + // Indexes of the weak imported files in the dependency list. + // For Google-internal migration only. Do not use. + repeated int32 weak_dependency = 11; + + // All top-level definitions in this file. + repeated DescriptorProto message_type = 4; + repeated EnumDescriptorProto enum_type = 5; + repeated ServiceDescriptorProto service = 6; + repeated FieldDescriptorProto extension = 7; + + optional FileOptions options = 8; + + // This field contains optional information about the original source code. + // You may safely remove this entire field without harming runtime + // functionality of the descriptors -- the information is needed only by + // development tools. + optional SourceCodeInfo source_code_info = 9; + + // The syntax of the proto file. + // The supported values are "proto2" and "proto3". + optional string syntax = 12; +} + +// Describes a message type. +message DescriptorProto { + optional string name = 1; + + repeated FieldDescriptorProto field = 2; + repeated FieldDescriptorProto extension = 6; + + repeated DescriptorProto nested_type = 3; + repeated EnumDescriptorProto enum_type = 4; + + message ExtensionRange { + optional int32 start = 1; // Inclusive. + optional int32 end = 2; // Exclusive. + + optional ExtensionRangeOptions options = 3; + } + repeated ExtensionRange extension_range = 5; + + repeated OneofDescriptorProto oneof_decl = 8; + + optional MessageOptions options = 7; + + // Range of reserved tag numbers. Reserved tag numbers may not be used by + // fields or extension ranges in the same message. Reserved ranges may + // not overlap. + message ReservedRange { + optional int32 start = 1; // Inclusive. + optional int32 end = 2; // Exclusive. + } + repeated ReservedRange reserved_range = 9; + // Reserved field names, which may not be used by fields in the same message. + // A given name may only be reserved once. + repeated string reserved_name = 10; +} + +message ExtensionRangeOptions { + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + +// Describes a field within a message. +message FieldDescriptorProto { + enum Type { + // 0 is reserved for errors. + // Order is weird for historical reasons. + TYPE_DOUBLE = 1; + TYPE_FLOAT = 2; + // Not ZigZag encoded. Negative numbers take 10 bytes. Use TYPE_SINT64 if + // negative values are likely. + TYPE_INT64 = 3; + TYPE_UINT64 = 4; + // Not ZigZag encoded. Negative numbers take 10 bytes. Use TYPE_SINT32 if + // negative values are likely. + TYPE_INT32 = 5; + TYPE_FIXED64 = 6; + TYPE_FIXED32 = 7; + TYPE_BOOL = 8; + TYPE_STRING = 9; + // Tag-delimited aggregate. + // Group type is deprecated and not supported in proto3. However, Proto3 + // implementations should still be able to parse the group wire format and + // treat group fields as unknown fields. + TYPE_GROUP = 10; + TYPE_MESSAGE = 11; // Length-delimited aggregate. + + // New in version 2. + TYPE_BYTES = 12; + TYPE_UINT32 = 13; + TYPE_ENUM = 14; + TYPE_SFIXED32 = 15; + TYPE_SFIXED64 = 16; + TYPE_SINT32 = 17; // Uses ZigZag encoding. + TYPE_SINT64 = 18; // Uses ZigZag encoding. + } + + enum Label { + // 0 is reserved for errors + LABEL_OPTIONAL = 1; + LABEL_REQUIRED = 2; + LABEL_REPEATED = 3; + } + + optional string name = 1; + optional int32 number = 3; + optional Label label = 4; + + // If type_name is set, this need not be set. If both this and type_name + // are set, this must be one of TYPE_ENUM, TYPE_MESSAGE or TYPE_GROUP. + optional Type type = 5; + + // For message and enum types, this is the name of the type. If the name + // starts with a '.', it is fully-qualified. Otherwise, C++-like scoping + // rules are used to find the type (i.e. first the nested types within this + // message are searched, then within the parent, on up to the root + // namespace). + optional string type_name = 6; + + // For extensions, this is the name of the type being extended. It is + // resolved in the same manner as type_name. + optional string extendee = 2; + + // For numeric types, contains the original text representation of the value. + // For booleans, "true" or "false". + // For strings, contains the default text contents (not escaped in any way). + // For bytes, contains the C escaped value. All bytes >= 128 are escaped. + // TODO(kenton): Base-64 encode? + optional string default_value = 7; + + // If set, gives the index of a oneof in the containing type's oneof_decl + // list. This field is a member of that oneof. + optional int32 oneof_index = 9; + + // JSON name of this field. The value is set by protocol compiler. If the + // user has set a "json_name" option on this field, that option's value + // will be used. Otherwise, it's deduced from the field's name by converting + // it to camelCase. + optional string json_name = 10; + + optional FieldOptions options = 8; + + // If true, this is a proto3 "optional". When a proto3 field is optional, it + // tracks presence regardless of field type. + // + // When proto3_optional is true, this field must be belong to a oneof to + // signal to old proto3 clients that presence is tracked for this field. This + // oneof is known as a "synthetic" oneof, and this field must be its sole + // member (each proto3 optional field gets its own synthetic oneof). Synthetic + // oneofs exist in the descriptor only, and do not generate any API. Synthetic + // oneofs must be ordered after all "real" oneofs. + // + // For message fields, proto3_optional doesn't create any semantic change, + // since non-repeated message fields always track presence. However it still + // indicates the semantic detail of whether the user wrote "optional" or not. + // This can be useful for round-tripping the .proto file. For consistency we + // give message fields a synthetic oneof also, even though it is not required + // to track presence. This is especially important because the parser can't + // tell if a field is a message or an enum, so it must always create a + // synthetic oneof. + // + // Proto2 optional fields do not set this flag, because they already indicate + // optional with `LABEL_OPTIONAL`. + optional bool proto3_optional = 17; +} + +// Describes a oneof. +message OneofDescriptorProto { + optional string name = 1; + optional OneofOptions options = 2; +} + +// Describes an enum type. +message EnumDescriptorProto { + optional string name = 1; + + repeated EnumValueDescriptorProto value = 2; + + optional EnumOptions options = 3; + + // Range of reserved numeric values. Reserved values may not be used by + // entries in the same enum. Reserved ranges may not overlap. + // + // Note that this is distinct from DescriptorProto.ReservedRange in that it + // is inclusive such that it can appropriately represent the entire int32 + // domain. + message EnumReservedRange { + optional int32 start = 1; // Inclusive. + optional int32 end = 2; // Inclusive. + } + + // Range of reserved numeric values. Reserved numeric values may not be used + // by enum values in the same enum declaration. Reserved ranges may not + // overlap. + repeated EnumReservedRange reserved_range = 4; + + // Reserved enum value names, which may not be reused. A given name may only + // be reserved once. + repeated string reserved_name = 5; +} + +// Describes a value within an enum. +message EnumValueDescriptorProto { + optional string name = 1; + optional int32 number = 2; + + optional EnumValueOptions options = 3; +} + +// Describes a service. +message ServiceDescriptorProto { + optional string name = 1; + repeated MethodDescriptorProto method = 2; + + optional ServiceOptions options = 3; +} + +// Describes a method of a service. +message MethodDescriptorProto { + optional string name = 1; + + // Input and output type names. These are resolved in the same way as + // FieldDescriptorProto.type_name, but must refer to a message type. + optional string input_type = 2; + optional string output_type = 3; + + optional MethodOptions options = 4; + + // Identifies if client streams multiple client messages + optional bool client_streaming = 5 [default = false]; + // Identifies if server streams multiple server messages + optional bool server_streaming = 6 [default = false]; +} + + +// =================================================================== +// Options + +// Each of the definitions above may have "options" attached. These are +// just annotations which may cause code to be generated slightly differently +// or may contain hints for code that manipulates protocol messages. +// +// Clients may define custom options as extensions of the *Options messages. +// These extensions may not yet be known at parsing time, so the parser cannot +// store the values in them. Instead it stores them in a field in the *Options +// message called uninterpreted_option. This field must have the same name +// across all *Options messages. We then use this field to populate the +// extensions when we build a descriptor, at which point all protos have been +// parsed and so all extensions are known. +// +// Extension numbers for custom options may be chosen as follows: +// * For options which will only be used within a single application or +// organization, or for experimental options, use field numbers 50000 +// through 99999. It is up to you to ensure that you do not use the +// same number for multiple options. +// * For options which will be published and used publicly by multiple +// independent entities, e-mail protobuf-global-extension-registry@google.com +// to reserve extension numbers. Simply provide your project name (e.g. +// Objective-C plugin) and your project website (if available) -- there's no +// need to explain how you intend to use them. Usually you only need one +// extension number. You can declare multiple options with only one extension +// number by putting them in a sub-message. See the Custom Options section of +// the docs for examples: +// https://developers.google.com/protocol-buffers/docs/proto#options +// If this turns out to be popular, a web service will be set up +// to automatically assign option numbers. + +message FileOptions { + + // Sets the Java package where classes generated from this .proto will be + // placed. By default, the proto package is used, but this is often + // inappropriate because proto packages do not normally start with backwards + // domain names. + optional string java_package = 1; + + + // Controls the name of the wrapper Java class generated for the .proto file. + // That class will always contain the .proto file's getDescriptor() method as + // well as any top-level extensions defined in the .proto file. + // If java_multiple_files is disabled, then all the other classes from the + // .proto file will be nested inside the single wrapper outer class. + optional string java_outer_classname = 8; + + // If enabled, then the Java code generator will generate a separate .java + // file for each top-level message, enum, and service defined in the .proto + // file. Thus, these types will *not* be nested inside the wrapper class + // named by java_outer_classname. However, the wrapper class will still be + // generated to contain the file's getDescriptor() method as well as any + // top-level extensions defined in the file. + optional bool java_multiple_files = 10 [default = false]; + + // This option does nothing. + optional bool java_generate_equals_and_hash = 20 [deprecated=true]; + + // If set true, then the Java2 code generator will generate code that + // throws an exception whenever an attempt is made to assign a non-UTF-8 + // byte sequence to a string field. + // Message reflection will do the same. + // However, an extension field still accepts non-UTF-8 byte sequences. + // This option has no effect on when used with the lite runtime. + optional bool java_string_check_utf8 = 27 [default = false]; + + + // Generated classes can be optimized for speed or code size. + enum OptimizeMode { + SPEED = 1; // Generate complete code for parsing, serialization, + // etc. + CODE_SIZE = 2; // Use ReflectionOps to implement these methods. + LITE_RUNTIME = 3; // Generate code using MessageLite and the lite runtime. + } + optional OptimizeMode optimize_for = 9 [default = SPEED]; + + // Sets the Go package where structs generated from this .proto will be + // placed. If omitted, the Go package will be derived from the following: + // - The basename of the package import path, if provided. + // - Otherwise, the package statement in the .proto file, if present. + // - Otherwise, the basename of the .proto file, without extension. + optional string go_package = 11; + + + + + // Should generic services be generated in each language? "Generic" services + // are not specific to any particular RPC system. They are generated by the + // main code generators in each language (without additional plugins). + // Generic services were the only kind of service generation supported by + // early versions of google.protobuf. + // + // Generic services are now considered deprecated in favor of using plugins + // that generate code specific to your particular RPC system. Therefore, + // these default to false. Old code which depends on generic services should + // explicitly set them to true. + optional bool cc_generic_services = 16 [default = false]; + optional bool java_generic_services = 17 [default = false]; + optional bool py_generic_services = 18 [default = false]; + optional bool php_generic_services = 42 [default = false]; + + // Is this file deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for everything in the file, or it will be completely ignored; in the very + // least, this is a formalization for deprecating files. + optional bool deprecated = 23 [default = false]; + + // Enables the use of arenas for the proto messages in this file. This applies + // only to generated classes for C++. + optional bool cc_enable_arenas = 31 [default = true]; + + + // Sets the objective c class prefix which is prepended to all objective c + // generated classes from this .proto. There is no default. + optional string objc_class_prefix = 36; + + // Namespace for generated classes; defaults to the package. + optional string csharp_namespace = 37; + + // By default Swift generators will take the proto package and CamelCase it + // replacing '.' with underscore and use that to prefix the types/symbols + // defined. When this options is provided, they will use this value instead + // to prefix the types/symbols defined. + optional string swift_prefix = 39; + + // Sets the php class prefix which is prepended to all php generated classes + // from this .proto. Default is empty. + optional string php_class_prefix = 40; + + // Use this option to change the namespace of php generated classes. Default + // is empty. When this option is empty, the package name will be used for + // determining the namespace. + optional string php_namespace = 41; + + // Use this option to change the namespace of php generated metadata classes. + // Default is empty. When this option is empty, the proto file name will be + // used for determining the namespace. + optional string php_metadata_namespace = 44; + + // Use this option to change the package of ruby generated classes. Default + // is empty. When this option is not set, the package name will be used for + // determining the ruby package. + optional string ruby_package = 45; + + + // The parser stores options it doesn't recognize here. + // See the documentation for the "Options" section above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. + // See the documentation for the "Options" section above. + extensions 1000 to max; + + reserved 38; +} + +message MessageOptions { + // Set true to use the old proto1 MessageSet wire format for extensions. + // This is provided for backwards-compatibility with the MessageSet wire + // format. You should not use this for any other reason: It's less + // efficient, has fewer features, and is more complicated. + // + // The message must be defined exactly as follows: + // message Foo { + // option message_set_wire_format = true; + // extensions 4 to max; + // } + // Note that the message cannot have any defined fields; MessageSets only + // have extensions. + // + // All extensions of your type must be singular messages; e.g. they cannot + // be int32s, enums, or repeated messages. + // + // Because this is an option, the above two restrictions are not enforced by + // the protocol compiler. + optional bool message_set_wire_format = 1 [default = false]; + + // Disables the generation of the standard "descriptor()" accessor, which can + // conflict with a field of the same name. This is meant to make migration + // from proto1 easier; new code should avoid fields named "descriptor". + optional bool no_standard_descriptor_accessor = 2 [default = false]; + + // Is this message deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for the message, or it will be completely ignored; in the very least, + // this is a formalization for deprecating messages. + optional bool deprecated = 3 [default = false]; + + reserved 4, 5, 6; + + // Whether the message is an automatically generated map entry type for the + // maps field. + // + // For maps fields: + // map map_field = 1; + // The parsed descriptor looks like: + // message MapFieldEntry { + // option map_entry = true; + // optional KeyType key = 1; + // optional ValueType value = 2; + // } + // repeated MapFieldEntry map_field = 1; + // + // Implementations may choose not to generate the map_entry=true message, but + // use a native map in the target language to hold the keys and values. + // The reflection APIs in such implementations still need to work as + // if the field is a repeated message field. + // + // NOTE: Do not set the option in .proto files. Always use the maps syntax + // instead. The option should only be implicitly set by the proto compiler + // parser. + optional bool map_entry = 7; + + reserved 8; // javalite_serializable + reserved 9; // javanano_as_lite + + + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + +message FieldOptions { + // The ctype option instructs the C++ code generator to use a different + // representation of the field than it normally would. See the specific + // options below. This option is not yet implemented in the open source + // release -- sorry, we'll try to include it in a future version! + optional CType ctype = 1 [default = STRING]; + enum CType { + // Default mode. + STRING = 0; + + CORD = 1; + + STRING_PIECE = 2; + } + // The packed option can be enabled for repeated primitive fields to enable + // a more efficient representation on the wire. Rather than repeatedly + // writing the tag and type for each element, the entire array is encoded as + // a single length-delimited blob. In proto3, only explicit setting it to + // false will avoid using packed encoding. + optional bool packed = 2; + + // The jstype option determines the JavaScript type used for values of the + // field. The option is permitted only for 64 bit integral and fixed types + // (int64, uint64, sint64, fixed64, sfixed64). A field with jstype JS_STRING + // is represented as JavaScript string, which avoids loss of precision that + // can happen when a large value is converted to a floating point JavaScript. + // Specifying JS_NUMBER for the jstype causes the generated JavaScript code to + // use the JavaScript "number" type. The behavior of the default option + // JS_NORMAL is implementation dependent. + // + // This option is an enum to permit additional types to be added, e.g. + // goog.math.Integer. + optional JSType jstype = 6 [default = JS_NORMAL]; + enum JSType { + // Use the default type. + JS_NORMAL = 0; + + // Use JavaScript strings. + JS_STRING = 1; + + // Use JavaScript numbers. + JS_NUMBER = 2; + } + + // Should this field be parsed lazily? Lazy applies only to message-type + // fields. It means that when the outer message is initially parsed, the + // inner message's contents will not be parsed but instead stored in encoded + // form. The inner message will actually be parsed when it is first accessed. + // + // This is only a hint. Implementations are free to choose whether to use + // eager or lazy parsing regardless of the value of this option. However, + // setting this option true suggests that the protocol author believes that + // using lazy parsing on this field is worth the additional bookkeeping + // overhead typically needed to implement it. + // + // This option does not affect the public interface of any generated code; + // all method signatures remain the same. Furthermore, thread-safety of the + // interface is not affected by this option; const methods remain safe to + // call from multiple threads concurrently, while non-const methods continue + // to require exclusive access. + // + // + // Note that implementations may choose not to check required fields within + // a lazy sub-message. That is, calling IsInitialized() on the outer message + // may return true even if the inner message has missing required fields. + // This is necessary because otherwise the inner message would have to be + // parsed in order to perform the check, defeating the purpose of lazy + // parsing. An implementation which chooses not to check required fields + // must be consistent about it. That is, for any particular sub-message, the + // implementation must either *always* check its required fields, or *never* + // check its required fields, regardless of whether or not the message has + // been parsed. + optional bool lazy = 5 [default = false]; + + // Is this field deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for accessors, or it will be completely ignored; in the very least, this + // is a formalization for deprecating fields. + optional bool deprecated = 3 [default = false]; + + // For Google-internal migration only. Do not use. + optional bool weak = 10 [default = false]; + + + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; + + reserved 4; // removed jtype +} + +message OneofOptions { + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + +message EnumOptions { + + // Set this option to true to allow mapping different tag names to the same + // value. + optional bool allow_alias = 2; + + // Is this enum deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for the enum, or it will be completely ignored; in the very least, this + // is a formalization for deprecating enums. + optional bool deprecated = 3 [default = false]; + + reserved 5; // javanano_as_lite + + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + +message EnumValueOptions { + // Is this enum value deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for the enum value, or it will be completely ignored; in the very least, + // this is a formalization for deprecating enum values. + optional bool deprecated = 1 [default = false]; + + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + +message ServiceOptions { + + // Note: Field numbers 1 through 32 are reserved for Google's internal RPC + // framework. We apologize for hoarding these numbers to ourselves, but + // we were already using them long before we decided to release Protocol + // Buffers. + + // Is this service deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for the service, or it will be completely ignored; in the very least, + // this is a formalization for deprecating services. + optional bool deprecated = 33 [default = false]; + + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + +message MethodOptions { + + // Note: Field numbers 1 through 32 are reserved for Google's internal RPC + // framework. We apologize for hoarding these numbers to ourselves, but + // we were already using them long before we decided to release Protocol + // Buffers. + + // Is this method deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for the method, or it will be completely ignored; in the very least, + // this is a formalization for deprecating methods. + optional bool deprecated = 33 [default = false]; + + // Is this method side-effect-free (or safe in HTTP parlance), or idempotent, + // or neither? HTTP based RPC implementation may choose GET verb for safe + // methods, and PUT verb for idempotent methods instead of the default POST. + enum IdempotencyLevel { + IDEMPOTENCY_UNKNOWN = 0; + NO_SIDE_EFFECTS = 1; // implies idempotent + IDEMPOTENT = 2; // idempotent, but may have side effects + } + optional IdempotencyLevel idempotency_level = 34 + [default = IDEMPOTENCY_UNKNOWN]; + + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + + +// A message representing a option the parser does not recognize. This only +// appears in options protos created by the compiler::Parser class. +// DescriptorPool resolves these when building Descriptor objects. Therefore, +// options protos in descriptor objects (e.g. returned by Descriptor::options(), +// or produced by Descriptor::CopyTo()) will never have UninterpretedOptions +// in them. +message UninterpretedOption { + // The name of the uninterpreted option. Each string represents a segment in + // a dot-separated name. is_extension is true iff a segment represents an + // extension (denoted with parentheses in options specs in .proto files). + // E.g.,{ ["foo", false], ["bar.baz", true], ["qux", false] } represents + // "foo.(bar.baz).qux". + message NamePart { + required string name_part = 1; + required bool is_extension = 2; + } + repeated NamePart name = 2; + + // The value of the uninterpreted option, in whatever type the tokenizer + // identified it as during parsing. Exactly one of these should be set. + optional string identifier_value = 3; + optional uint64 positive_int_value = 4; + optional int64 negative_int_value = 5; + optional double double_value = 6; + optional bytes string_value = 7; + optional string aggregate_value = 8; +} + +// =================================================================== +// Optional source code info + +// Encapsulates information about the original source file from which a +// FileDescriptorProto was generated. +message SourceCodeInfo { + // A Location identifies a piece of source code in a .proto file which + // corresponds to a particular definition. This information is intended + // to be useful to IDEs, code indexers, documentation generators, and similar + // tools. + // + // For example, say we have a file like: + // message Foo { + // optional string foo = 1; + // } + // Let's look at just the field definition: + // optional string foo = 1; + // ^ ^^ ^^ ^ ^^^ + // a bc de f ghi + // We have the following locations: + // span path represents + // [a,i) [ 4, 0, 2, 0 ] The whole field definition. + // [a,b) [ 4, 0, 2, 0, 4 ] The label (optional). + // [c,d) [ 4, 0, 2, 0, 5 ] The type (string). + // [e,f) [ 4, 0, 2, 0, 1 ] The name (foo). + // [g,h) [ 4, 0, 2, 0, 3 ] The number (1). + // + // Notes: + // - A location may refer to a repeated field itself (i.e. not to any + // particular index within it). This is used whenever a set of elements are + // logically enclosed in a single code segment. For example, an entire + // extend block (possibly containing multiple extension definitions) will + // have an outer location whose path refers to the "extensions" repeated + // field without an index. + // - Multiple locations may have the same path. This happens when a single + // logical declaration is spread out across multiple places. The most + // obvious example is the "extend" block again -- there may be multiple + // extend blocks in the same scope, each of which will have the same path. + // - A location's span is not always a subset of its parent's span. For + // example, the "extendee" of an extension declaration appears at the + // beginning of the "extend" block and is shared by all extensions within + // the block. + // - Just because a location's span is a subset of some other location's span + // does not mean that it is a descendant. For example, a "group" defines + // both a type and a field in a single declaration. Thus, the locations + // corresponding to the type and field and their components will overlap. + // - Code which tries to interpret locations should probably be designed to + // ignore those that it doesn't understand, as more types of locations could + // be recorded in the future. + repeated Location location = 1; + message Location { + // Identifies which part of the FileDescriptorProto was defined at this + // location. + // + // Each element is a field number or an index. They form a path from + // the root FileDescriptorProto to the place where the definition. For + // example, this path: + // [ 4, 3, 2, 7, 1 ] + // refers to: + // file.message_type(3) // 4, 3 + // .field(7) // 2, 7 + // .name() // 1 + // This is because FileDescriptorProto.message_type has field number 4: + // repeated DescriptorProto message_type = 4; + // and DescriptorProto.field has field number 2: + // repeated FieldDescriptorProto field = 2; + // and FieldDescriptorProto.name has field number 1: + // optional string name = 1; + // + // Thus, the above path gives the location of a field name. If we removed + // the last element: + // [ 4, 3, 2, 7 ] + // this path refers to the whole field declaration (from the beginning + // of the label to the terminating semicolon). + repeated int32 path = 1 [packed = true]; + + // Always has exactly three or four elements: start line, start column, + // end line (optional, otherwise assumed same as start line), end column. + // These are packed into a single field for efficiency. Note that line + // and column numbers are zero-based -- typically you will want to add + // 1 to each before displaying to a user. + repeated int32 span = 2 [packed = true]; + + // If this SourceCodeInfo represents a complete declaration, these are any + // comments appearing before and after the declaration which appear to be + // attached to the declaration. + // + // A series of line comments appearing on consecutive lines, with no other + // tokens appearing on those lines, will be treated as a single comment. + // + // leading_detached_comments will keep paragraphs of comments that appear + // before (but not connected to) the current element. Each paragraph, + // separated by empty lines, will be one comment element in the repeated + // field. + // + // Only the comment content is provided; comment markers (e.g. //) are + // stripped out. For block comments, leading whitespace and an asterisk + // will be stripped from the beginning of each line other than the first. + // Newlines are included in the output. + // + // Examples: + // + // optional int32 foo = 1; // Comment attached to foo. + // // Comment attached to bar. + // optional int32 bar = 2; + // + // optional string baz = 3; + // // Comment attached to baz. + // // Another line attached to baz. + // + // // Comment attached to qux. + // // + // // Another line attached to qux. + // optional double qux = 4; + // + // // Detached comment for corge. This is not leading or trailing comments + // // to qux or corge because there are blank lines separating it from + // // both. + // + // // Detached comment for corge paragraph 2. + // + // optional string corge = 5; + // /* Block comment attached + // * to corge. Leading asterisks + // * will be removed. */ + // /* Block comment attached to + // * grault. */ + // optional int32 grault = 6; + // + // // ignored detached comments. + optional string leading_comments = 3; + optional string trailing_comments = 4; + repeated string leading_detached_comments = 6; + } +} + +// Describes the relationship between generated code and its original source +// file. A GeneratedCodeInfo message is associated with only one generated +// source file, but may contain references to different source .proto files. +message GeneratedCodeInfo { + // An Annotation connects some span of text in generated code to an element + // of its generating .proto file. + repeated Annotation annotation = 1; + message Annotation { + // Identifies the element in the original source .proto file. This field + // is formatted the same as SourceCodeInfo.Location.path. + repeated int32 path = 1 [packed = true]; + + // Identifies the filesystem path to the original source .proto. + optional string source_file = 2; + + // Identifies the starting offset in bytes in the generated code + // that relates to the identified object. + optional int32 begin = 3; + + // Identifies the ending offset in bytes in the generated code that + // relates to the identified offset. The end offset should be one past + // the last relevant byte (so the length of the text = end - begin). + optional int32 end = 4; + } +} diff --git a/05_sparkplugb/protobuf/include/google/protobuf/duration.proto b/05_sparkplugb/protobuf/include/google/protobuf/duration.proto new file mode 100644 index 00000000..81c3e369 --- /dev/null +++ b/05_sparkplugb/protobuf/include/google/protobuf/duration.proto @@ -0,0 +1,116 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option cc_enable_arenas = true; +option go_package = "google.golang.org/protobuf/types/known/durationpb"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "DurationProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; + +// A Duration represents a signed, fixed-length span of time represented +// as a count of seconds and fractions of seconds at nanosecond +// resolution. It is independent of any calendar and concepts like "day" +// or "month". It is related to Timestamp in that the difference between +// two Timestamp values is a Duration and it can be added or subtracted +// from a Timestamp. Range is approximately +-10,000 years. +// +// # Examples +// +// Example 1: Compute Duration from two Timestamps in pseudo code. +// +// Timestamp start = ...; +// Timestamp end = ...; +// Duration duration = ...; +// +// duration.seconds = end.seconds - start.seconds; +// duration.nanos = end.nanos - start.nanos; +// +// if (duration.seconds < 0 && duration.nanos > 0) { +// duration.seconds += 1; +// duration.nanos -= 1000000000; +// } else if (duration.seconds > 0 && duration.nanos < 0) { +// duration.seconds -= 1; +// duration.nanos += 1000000000; +// } +// +// Example 2: Compute Timestamp from Timestamp + Duration in pseudo code. +// +// Timestamp start = ...; +// Duration duration = ...; +// Timestamp end = ...; +// +// end.seconds = start.seconds + duration.seconds; +// end.nanos = start.nanos + duration.nanos; +// +// if (end.nanos < 0) { +// end.seconds -= 1; +// end.nanos += 1000000000; +// } else if (end.nanos >= 1000000000) { +// end.seconds += 1; +// end.nanos -= 1000000000; +// } +// +// Example 3: Compute Duration from datetime.timedelta in Python. +// +// td = datetime.timedelta(days=3, minutes=10) +// duration = Duration() +// duration.FromTimedelta(td) +// +// # JSON Mapping +// +// In JSON format, the Duration type is encoded as a string rather than an +// object, where the string ends in the suffix "s" (indicating seconds) and +// is preceded by the number of seconds, with nanoseconds expressed as +// fractional seconds. For example, 3 seconds with 0 nanoseconds should be +// encoded in JSON format as "3s", while 3 seconds and 1 nanosecond should +// be expressed in JSON format as "3.000000001s", and 3 seconds and 1 +// microsecond should be expressed in JSON format as "3.000001s". +// +// +message Duration { + // Signed seconds of the span of time. Must be from -315,576,000,000 + // to +315,576,000,000 inclusive. Note: these bounds are computed from: + // 60 sec/min * 60 min/hr * 24 hr/day * 365.25 days/year * 10000 years + int64 seconds = 1; + + // Signed fractions of a second at nanosecond resolution of the span + // of time. Durations less than one second are represented with a 0 + // `seconds` field and a positive or negative `nanos` field. For durations + // of one second or more, a non-zero value for the `nanos` field must be + // of the same sign as the `seconds` field. Must be from -999,999,999 + // to +999,999,999 inclusive. + int32 nanos = 2; +} diff --git a/05_sparkplugb/protobuf/include/google/protobuf/empty.proto b/05_sparkplugb/protobuf/include/google/protobuf/empty.proto new file mode 100644 index 00000000..5f992de9 --- /dev/null +++ b/05_sparkplugb/protobuf/include/google/protobuf/empty.proto @@ -0,0 +1,52 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option go_package = "google.golang.org/protobuf/types/known/emptypb"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "EmptyProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; +option cc_enable_arenas = true; + +// A generic empty message that you can re-use to avoid defining duplicated +// empty messages in your APIs. A typical example is to use it as the request +// or the response type of an API method. For instance: +// +// service Foo { +// rpc Bar(google.protobuf.Empty) returns (google.protobuf.Empty); +// } +// +// The JSON representation for `Empty` is empty JSON object `{}`. +message Empty {} diff --git a/05_sparkplugb/protobuf/include/google/protobuf/field_mask.proto b/05_sparkplugb/protobuf/include/google/protobuf/field_mask.proto new file mode 100644 index 00000000..6b5104f1 --- /dev/null +++ b/05_sparkplugb/protobuf/include/google/protobuf/field_mask.proto @@ -0,0 +1,245 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "FieldMaskProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; +option go_package = "google.golang.org/protobuf/types/known/fieldmaskpb"; +option cc_enable_arenas = true; + +// `FieldMask` represents a set of symbolic field paths, for example: +// +// paths: "f.a" +// paths: "f.b.d" +// +// Here `f` represents a field in some root message, `a` and `b` +// fields in the message found in `f`, and `d` a field found in the +// message in `f.b`. +// +// Field masks are used to specify a subset of fields that should be +// returned by a get operation or modified by an update operation. +// Field masks also have a custom JSON encoding (see below). +// +// # Field Masks in Projections +// +// When used in the context of a projection, a response message or +// sub-message is filtered by the API to only contain those fields as +// specified in the mask. For example, if the mask in the previous +// example is applied to a response message as follows: +// +// f { +// a : 22 +// b { +// d : 1 +// x : 2 +// } +// y : 13 +// } +// z: 8 +// +// The result will not contain specific values for fields x,y and z +// (their value will be set to the default, and omitted in proto text +// output): +// +// +// f { +// a : 22 +// b { +// d : 1 +// } +// } +// +// A repeated field is not allowed except at the last position of a +// paths string. +// +// If a FieldMask object is not present in a get operation, the +// operation applies to all fields (as if a FieldMask of all fields +// had been specified). +// +// Note that a field mask does not necessarily apply to the +// top-level response message. In case of a REST get operation, the +// field mask applies directly to the response, but in case of a REST +// list operation, the mask instead applies to each individual message +// in the returned resource list. In case of a REST custom method, +// other definitions may be used. Where the mask applies will be +// clearly documented together with its declaration in the API. In +// any case, the effect on the returned resource/resources is required +// behavior for APIs. +// +// # Field Masks in Update Operations +// +// A field mask in update operations specifies which fields of the +// targeted resource are going to be updated. The API is required +// to only change the values of the fields as specified in the mask +// and leave the others untouched. If a resource is passed in to +// describe the updated values, the API ignores the values of all +// fields not covered by the mask. +// +// If a repeated field is specified for an update operation, new values will +// be appended to the existing repeated field in the target resource. Note that +// a repeated field is only allowed in the last position of a `paths` string. +// +// If a sub-message is specified in the last position of the field mask for an +// update operation, then new value will be merged into the existing sub-message +// in the target resource. +// +// For example, given the target message: +// +// f { +// b { +// d: 1 +// x: 2 +// } +// c: [1] +// } +// +// And an update message: +// +// f { +// b { +// d: 10 +// } +// c: [2] +// } +// +// then if the field mask is: +// +// paths: ["f.b", "f.c"] +// +// then the result will be: +// +// f { +// b { +// d: 10 +// x: 2 +// } +// c: [1, 2] +// } +// +// An implementation may provide options to override this default behavior for +// repeated and message fields. +// +// In order to reset a field's value to the default, the field must +// be in the mask and set to the default value in the provided resource. +// Hence, in order to reset all fields of a resource, provide a default +// instance of the resource and set all fields in the mask, or do +// not provide a mask as described below. +// +// If a field mask is not present on update, the operation applies to +// all fields (as if a field mask of all fields has been specified). +// Note that in the presence of schema evolution, this may mean that +// fields the client does not know and has therefore not filled into +// the request will be reset to their default. If this is unwanted +// behavior, a specific service may require a client to always specify +// a field mask, producing an error if not. +// +// As with get operations, the location of the resource which +// describes the updated values in the request message depends on the +// operation kind. In any case, the effect of the field mask is +// required to be honored by the API. +// +// ## Considerations for HTTP REST +// +// The HTTP kind of an update operation which uses a field mask must +// be set to PATCH instead of PUT in order to satisfy HTTP semantics +// (PUT must only be used for full updates). +// +// # JSON Encoding of Field Masks +// +// In JSON, a field mask is encoded as a single string where paths are +// separated by a comma. Fields name in each path are converted +// to/from lower-camel naming conventions. +// +// As an example, consider the following message declarations: +// +// message Profile { +// User user = 1; +// Photo photo = 2; +// } +// message User { +// string display_name = 1; +// string address = 2; +// } +// +// In proto a field mask for `Profile` may look as such: +// +// mask { +// paths: "user.display_name" +// paths: "photo" +// } +// +// In JSON, the same mask is represented as below: +// +// { +// mask: "user.displayName,photo" +// } +// +// # Field Masks and Oneof Fields +// +// Field masks treat fields in oneofs just as regular fields. Consider the +// following message: +// +// message SampleMessage { +// oneof test_oneof { +// string name = 4; +// SubMessage sub_message = 9; +// } +// } +// +// The field mask can be: +// +// mask { +// paths: "name" +// } +// +// Or: +// +// mask { +// paths: "sub_message" +// } +// +// Note that oneof type names ("test_oneof" in this case) cannot be used in +// paths. +// +// ## Field Mask Verification +// +// The implementation of any API method which has a FieldMask type field in the +// request should verify the included field paths, and return an +// `INVALID_ARGUMENT` error if any path is unmappable. +message FieldMask { + // The set of field mask paths. + repeated string paths = 1; +} diff --git a/05_sparkplugb/protobuf/include/google/protobuf/source_context.proto b/05_sparkplugb/protobuf/include/google/protobuf/source_context.proto new file mode 100644 index 00000000..06bfc43a --- /dev/null +++ b/05_sparkplugb/protobuf/include/google/protobuf/source_context.proto @@ -0,0 +1,48 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "SourceContextProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; +option go_package = "google.golang.org/protobuf/types/known/sourcecontextpb"; + +// `SourceContext` represents information about the source of a +// protobuf element, like the file in which it is defined. +message SourceContext { + // The path-qualified name of the .proto file that contained the associated + // protobuf element. For example: `"google/protobuf/source_context.proto"`. + string file_name = 1; +} diff --git a/05_sparkplugb/protobuf/include/google/protobuf/struct.proto b/05_sparkplugb/protobuf/include/google/protobuf/struct.proto new file mode 100644 index 00000000..0ac843ca --- /dev/null +++ b/05_sparkplugb/protobuf/include/google/protobuf/struct.proto @@ -0,0 +1,95 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option cc_enable_arenas = true; +option go_package = "google.golang.org/protobuf/types/known/structpb"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "StructProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; + +// `Struct` represents a structured data value, consisting of fields +// which map to dynamically typed values. In some languages, `Struct` +// might be supported by a native representation. For example, in +// scripting languages like JS a struct is represented as an +// object. The details of that representation are described together +// with the proto support for the language. +// +// The JSON representation for `Struct` is JSON object. +message Struct { + // Unordered map of dynamically typed values. + map fields = 1; +} + +// `Value` represents a dynamically typed value which can be either +// null, a number, a string, a boolean, a recursive struct value, or a +// list of values. A producer of value is expected to set one of these +// variants. Absence of any variant indicates an error. +// +// The JSON representation for `Value` is JSON value. +message Value { + // The kind of value. + oneof kind { + // Represents a null value. + NullValue null_value = 1; + // Represents a double value. + double number_value = 2; + // Represents a string value. + string string_value = 3; + // Represents a boolean value. + bool bool_value = 4; + // Represents a structured value. + Struct struct_value = 5; + // Represents a repeated `Value`. + ListValue list_value = 6; + } +} + +// `NullValue` is a singleton enumeration to represent the null value for the +// `Value` type union. +// +// The JSON representation for `NullValue` is JSON `null`. +enum NullValue { + // Null value. + NULL_VALUE = 0; +} + +// `ListValue` is a wrapper around a repeated field of values. +// +// The JSON representation for `ListValue` is JSON array. +message ListValue { + // Repeated field of dynamically typed values. + repeated Value values = 1; +} diff --git a/05_sparkplugb/protobuf/include/google/protobuf/timestamp.proto b/05_sparkplugb/protobuf/include/google/protobuf/timestamp.proto new file mode 100644 index 00000000..3b2df6d9 --- /dev/null +++ b/05_sparkplugb/protobuf/include/google/protobuf/timestamp.proto @@ -0,0 +1,147 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option cc_enable_arenas = true; +option go_package = "google.golang.org/protobuf/types/known/timestamppb"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "TimestampProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; + +// A Timestamp represents a point in time independent of any time zone or local +// calendar, encoded as a count of seconds and fractions of seconds at +// nanosecond resolution. The count is relative to an epoch at UTC midnight on +// January 1, 1970, in the proleptic Gregorian calendar which extends the +// Gregorian calendar backwards to year one. +// +// All minutes are 60 seconds long. Leap seconds are "smeared" so that no leap +// second table is needed for interpretation, using a [24-hour linear +// smear](https://developers.google.com/time/smear). +// +// The range is from 0001-01-01T00:00:00Z to 9999-12-31T23:59:59.999999999Z. By +// restricting to that range, we ensure that we can convert to and from [RFC +// 3339](https://www.ietf.org/rfc/rfc3339.txt) date strings. +// +// # Examples +// +// Example 1: Compute Timestamp from POSIX `time()`. +// +// Timestamp timestamp; +// timestamp.set_seconds(time(NULL)); +// timestamp.set_nanos(0); +// +// Example 2: Compute Timestamp from POSIX `gettimeofday()`. +// +// struct timeval tv; +// gettimeofday(&tv, NULL); +// +// Timestamp timestamp; +// timestamp.set_seconds(tv.tv_sec); +// timestamp.set_nanos(tv.tv_usec * 1000); +// +// Example 3: Compute Timestamp from Win32 `GetSystemTimeAsFileTime()`. +// +// FILETIME ft; +// GetSystemTimeAsFileTime(&ft); +// UINT64 ticks = (((UINT64)ft.dwHighDateTime) << 32) | ft.dwLowDateTime; +// +// // A Windows tick is 100 nanoseconds. Windows epoch 1601-01-01T00:00:00Z +// // is 11644473600 seconds before Unix epoch 1970-01-01T00:00:00Z. +// Timestamp timestamp; +// timestamp.set_seconds((INT64) ((ticks / 10000000) - 11644473600LL)); +// timestamp.set_nanos((INT32) ((ticks % 10000000) * 100)); +// +// Example 4: Compute Timestamp from Java `System.currentTimeMillis()`. +// +// long millis = System.currentTimeMillis(); +// +// Timestamp timestamp = Timestamp.newBuilder().setSeconds(millis / 1000) +// .setNanos((int) ((millis % 1000) * 1000000)).build(); +// +// +// Example 5: Compute Timestamp from Java `Instant.now()`. +// +// Instant now = Instant.now(); +// +// Timestamp timestamp = +// Timestamp.newBuilder().setSeconds(now.getEpochSecond()) +// .setNanos(now.getNano()).build(); +// +// +// Example 6: Compute Timestamp from current time in Python. +// +// timestamp = Timestamp() +// timestamp.GetCurrentTime() +// +// # JSON Mapping +// +// In JSON format, the Timestamp type is encoded as a string in the +// [RFC 3339](https://www.ietf.org/rfc/rfc3339.txt) format. That is, the +// format is "{year}-{month}-{day}T{hour}:{min}:{sec}[.{frac_sec}]Z" +// where {year} is always expressed using four digits while {month}, {day}, +// {hour}, {min}, and {sec} are zero-padded to two digits each. The fractional +// seconds, which can go up to 9 digits (i.e. up to 1 nanosecond resolution), +// are optional. The "Z" suffix indicates the timezone ("UTC"); the timezone +// is required. A proto3 JSON serializer should always use UTC (as indicated by +// "Z") when printing the Timestamp type and a proto3 JSON parser should be +// able to accept both UTC and other timezones (as indicated by an offset). +// +// For example, "2017-01-15T01:30:15.01Z" encodes 15.01 seconds past +// 01:30 UTC on January 15, 2017. +// +// In JavaScript, one can convert a Date object to this format using the +// standard +// [toISOString()](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Date/toISOString) +// method. In Python, a standard `datetime.datetime` object can be converted +// to this format using +// [`strftime`](https://docs.python.org/2/library/time.html#time.strftime) with +// the time format spec '%Y-%m-%dT%H:%M:%S.%fZ'. Likewise, in Java, one can use +// the Joda Time's [`ISODateTimeFormat.dateTime()`]( +// http://www.joda.org/joda-time/apidocs/org/joda/time/format/ISODateTimeFormat.html#dateTime%2D%2D +// ) to obtain a formatter capable of generating timestamps in this format. +// +// +message Timestamp { + // Represents seconds of UTC time since Unix epoch + // 1970-01-01T00:00:00Z. Must be from 0001-01-01T00:00:00Z to + // 9999-12-31T23:59:59Z inclusive. + int64 seconds = 1; + + // Non-negative fractions of a second at nanosecond resolution. Negative + // second values with fractions must still have non-negative nanos values + // that count forward in time. Must be from 0 to 999,999,999 + // inclusive. + int32 nanos = 2; +} diff --git a/05_sparkplugb/protobuf/include/google/protobuf/type.proto b/05_sparkplugb/protobuf/include/google/protobuf/type.proto new file mode 100644 index 00000000..d3f6a68b --- /dev/null +++ b/05_sparkplugb/protobuf/include/google/protobuf/type.proto @@ -0,0 +1,187 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +import "google/protobuf/any.proto"; +import "google/protobuf/source_context.proto"; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option cc_enable_arenas = true; +option java_package = "com.google.protobuf"; +option java_outer_classname = "TypeProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; +option go_package = "google.golang.org/protobuf/types/known/typepb"; + +// A protocol buffer message type. +message Type { + // The fully qualified message name. + string name = 1; + // The list of fields. + repeated Field fields = 2; + // The list of types appearing in `oneof` definitions in this type. + repeated string oneofs = 3; + // The protocol buffer options. + repeated Option options = 4; + // The source context. + SourceContext source_context = 5; + // The source syntax. + Syntax syntax = 6; +} + +// A single field of a message type. +message Field { + // Basic field types. + enum Kind { + // Field type unknown. + TYPE_UNKNOWN = 0; + // Field type double. + TYPE_DOUBLE = 1; + // Field type float. + TYPE_FLOAT = 2; + // Field type int64. + TYPE_INT64 = 3; + // Field type uint64. + TYPE_UINT64 = 4; + // Field type int32. + TYPE_INT32 = 5; + // Field type fixed64. + TYPE_FIXED64 = 6; + // Field type fixed32. + TYPE_FIXED32 = 7; + // Field type bool. + TYPE_BOOL = 8; + // Field type string. + TYPE_STRING = 9; + // Field type group. Proto2 syntax only, and deprecated. + TYPE_GROUP = 10; + // Field type message. + TYPE_MESSAGE = 11; + // Field type bytes. + TYPE_BYTES = 12; + // Field type uint32. + TYPE_UINT32 = 13; + // Field type enum. + TYPE_ENUM = 14; + // Field type sfixed32. + TYPE_SFIXED32 = 15; + // Field type sfixed64. + TYPE_SFIXED64 = 16; + // Field type sint32. + TYPE_SINT32 = 17; + // Field type sint64. + TYPE_SINT64 = 18; + } + + // Whether a field is optional, required, or repeated. + enum Cardinality { + // For fields with unknown cardinality. + CARDINALITY_UNKNOWN = 0; + // For optional fields. + CARDINALITY_OPTIONAL = 1; + // For required fields. Proto2 syntax only. + CARDINALITY_REQUIRED = 2; + // For repeated fields. + CARDINALITY_REPEATED = 3; + } + + // The field type. + Kind kind = 1; + // The field cardinality. + Cardinality cardinality = 2; + // The field number. + int32 number = 3; + // The field name. + string name = 4; + // The field type URL, without the scheme, for message or enumeration + // types. Example: `"type.googleapis.com/google.protobuf.Timestamp"`. + string type_url = 6; + // The index of the field type in `Type.oneofs`, for message or enumeration + // types. The first type has index 1; zero means the type is not in the list. + int32 oneof_index = 7; + // Whether to use alternative packed wire representation. + bool packed = 8; + // The protocol buffer options. + repeated Option options = 9; + // The field JSON name. + string json_name = 10; + // The string value of the default value of this field. Proto2 syntax only. + string default_value = 11; +} + +// Enum type definition. +message Enum { + // Enum type name. + string name = 1; + // Enum value definitions. + repeated EnumValue enumvalue = 2; + // Protocol buffer options. + repeated Option options = 3; + // The source context. + SourceContext source_context = 4; + // The source syntax. + Syntax syntax = 5; +} + +// Enum value definition. +message EnumValue { + // Enum value name. + string name = 1; + // Enum value number. + int32 number = 2; + // Protocol buffer options. + repeated Option options = 3; +} + +// A protocol buffer option, which can be attached to a message, field, +// enumeration, etc. +message Option { + // The option's name. For protobuf built-in options (options defined in + // descriptor.proto), this is the short name. For example, `"map_entry"`. + // For custom options, it should be the fully-qualified name. For example, + // `"google.api.http"`. + string name = 1; + // The option's value packed in an Any message. If the value is a primitive, + // the corresponding wrapper type defined in google/protobuf/wrappers.proto + // should be used. If the value is an enum, it should be stored as an int32 + // value using the google.protobuf.Int32Value type. + Any value = 2; +} + +// The syntax in which a protocol buffer element is defined. +enum Syntax { + // Syntax `proto2`. + SYNTAX_PROTO2 = 0; + // Syntax `proto3`. + SYNTAX_PROTO3 = 1; +} diff --git a/05_sparkplugb/protobuf/include/google/protobuf/wrappers.proto b/05_sparkplugb/protobuf/include/google/protobuf/wrappers.proto new file mode 100644 index 00000000..d49dd53c --- /dev/null +++ b/05_sparkplugb/protobuf/include/google/protobuf/wrappers.proto @@ -0,0 +1,123 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Wrappers for primitive (non-message) types. These types are useful +// for embedding primitives in the `google.protobuf.Any` type and for places +// where we need to distinguish between the absence of a primitive +// typed field and its default value. +// +// These wrappers have no meaningful use within repeated fields as they lack +// the ability to detect presence on individual elements. +// These wrappers have no meaningful use within a map or a oneof since +// individual entries of a map or fields of a oneof can already detect presence. + +syntax = "proto3"; + +package google.protobuf; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option cc_enable_arenas = true; +option go_package = "google.golang.org/protobuf/types/known/wrapperspb"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "WrappersProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; + +// Wrapper message for `double`. +// +// The JSON representation for `DoubleValue` is JSON number. +message DoubleValue { + // The double value. + double value = 1; +} + +// Wrapper message for `float`. +// +// The JSON representation for `FloatValue` is JSON number. +message FloatValue { + // The float value. + float value = 1; +} + +// Wrapper message for `int64`. +// +// The JSON representation for `Int64Value` is JSON string. +message Int64Value { + // The int64 value. + int64 value = 1; +} + +// Wrapper message for `uint64`. +// +// The JSON representation for `UInt64Value` is JSON string. +message UInt64Value { + // The uint64 value. + uint64 value = 1; +} + +// Wrapper message for `int32`. +// +// The JSON representation for `Int32Value` is JSON number. +message Int32Value { + // The int32 value. + int32 value = 1; +} + +// Wrapper message for `uint32`. +// +// The JSON representation for `UInt32Value` is JSON number. +message UInt32Value { + // The uint32 value. + uint32 value = 1; +} + +// Wrapper message for `bool`. +// +// The JSON representation for `BoolValue` is JSON `true` and `false`. +message BoolValue { + // The bool value. + bool value = 1; +} + +// Wrapper message for `string`. +// +// The JSON representation for `StringValue` is JSON string. +message StringValue { + // The string value. + string value = 1; +} + +// Wrapper message for `bytes`. +// +// The JSON representation for `BytesValue` is JSON string. +message BytesValue { + // The bytes value. + bytes value = 1; +} diff --git a/05_sparkplugb/protobuf/readme.txt b/05_sparkplugb/protobuf/readme.txt new file mode 100644 index 00000000..b6c9f9b4 --- /dev/null +++ b/05_sparkplugb/protobuf/readme.txt @@ -0,0 +1,15 @@ +Protocol Buffers - Google's data interchange format +Copyright 2008 Google Inc. +https://developers.google.com/protocol-buffers/ + +This package contains a precompiled binary version of the protocol buffer +compiler (protoc). This binary is intended for users who want to use Protocol +Buffers in languages other than C++ but do not want to compile protoc +themselves. To install, simply place this binary somewhere in your PATH. + +If you intend to use the included well known types then don't forget to +copy the contents of the 'include' directory somewhere as well, for example +into '/usr/local/include/'. + +Please refer to our official github site for more installation instructions: + https://github.com/protocolbuffers/protobuf diff --git a/05_sparkplugb/requirements.txt b/05_sparkplugb/requirements.txt new file mode 100644 index 00000000..35a418ba --- /dev/null +++ b/05_sparkplugb/requirements.txt @@ -0,0 +1,4 @@ +wheel>=0.37.1 +paho-mqtt>=1.6.1 +protobuf>=4.21.1 +dynaconf>=3.1.9 diff --git a/05_sparkplugb/requirements_dev.txt b/05_sparkplugb/requirements_dev.txt new file mode 100644 index 00000000..5a8a8842 --- /dev/null +++ b/05_sparkplugb/requirements_dev.txt @@ -0,0 +1,2 @@ +pytest >=7.1.2 +flake8 >=4.0.1 \ No newline at end of file diff --git a/05_sparkplugb/setup.cfg b/05_sparkplugb/setup.cfg new file mode 100644 index 00000000..8204a3a4 --- /dev/null +++ b/05_sparkplugb/setup.cfg @@ -0,0 +1,21 @@ +[metadata] +name = uns_sparkplugb + +author = Ashwin Krishnan +author_email = mkashwin@gmail.com +description = SparkplugB decoder to parse and unpack SparkplugB messages to the UNS structure +long_description = file: README.md +long_description_content_type = text/markdown +classifiers = + Programming Language :: Python :: 3 + License :: OSI Approved :: MIT License + Operating System :: OS Independent + +[options] +package_dir = + = src +packages = find: +python_requires = >=3.10.5 + +[options.packages.find] +where = src \ No newline at end of file diff --git a/05_sparkplugb/setup.py b/05_sparkplugb/setup.py new file mode 100644 index 00000000..3367edfe --- /dev/null +++ b/05_sparkplugb/setup.py @@ -0,0 +1,4 @@ +from setuptools import setup, find_packages + +setup(name="uns_sparkplugb", + packages=find_packages(where="./src", exclude=("./tests"))) diff --git a/05_sparkplugb/sparkplug_b/sparkplug_b.proto b/05_sparkplugb/sparkplug_b/sparkplug_b.proto new file mode 100644 index 00000000..931c7992 --- /dev/null +++ b/05_sparkplugb/sparkplug_b/sparkplug_b.proto @@ -0,0 +1,224 @@ +// * Copyright (c) 2015, 2018 Cirrus Link Solutions and others +// * +// * This program and the accompanying materials are made available under the +// * terms of the Eclipse Public License 2.0 which is available at +// * http://www.eclipse.org/legal/epl-2.0. +// * +// * SPDX-License-Identifier: EPL-2.0 +// * +// * Contributors: +// * Cirrus Link Solutions - initial implementation + +// +// To compile: +// cd client_libraries/java +// protoc --proto_path=../../ --java_out=src/main/java ../../sparkplug_b.proto +// + +syntax = "proto2"; + +package org.eclipse.tahu.protobuf; + +option java_package = "org.eclipse.tahu.protobuf"; +option java_outer_classname = "SparkplugBProto"; + +enum DataType { + // Indexes of Data Types + + // Unknown placeholder for future expansion. + Unknown = 0; + + // Basic Types + Int8 = 1; + Int16 = 2; + Int32 = 3; + Int64 = 4; + UInt8 = 5; + UInt16 = 6; + UInt32 = 7; + UInt64 = 8; + Float = 9; + Double = 10; + Boolean = 11; + String = 12; + DateTime = 13; + Text = 14; + + // Additional Metric Types + UUID = 15; + DataSet = 16; + Bytes = 17; + File = 18; + Template = 19; + + // Additional PropertyValue Types + PropertySet = 20; + PropertySetList = 21; + + // Array Types + Int8Array = 22; + Int16Array = 23; + Int32Array = 24; + Int64Array = 25; + UInt8Array = 26; + UInt16Array = 27; + UInt32Array = 28; + UInt64Array = 29; + FloatArray = 30; + DoubleArray = 31; + BooleanArray = 32; + StringArray = 33; + DateTimeArray = 34; +} + +message Payload { + + message Template { + + message Parameter { + optional string name = 1; + optional uint32 type = 2; + + oneof value { + uint32 int_value = 3; + uint64 long_value = 4; + float float_value = 5; + double double_value = 6; + bool boolean_value = 7; + string string_value = 8; + ParameterValueExtension extension_value = 9; + } + + message ParameterValueExtension { + extensions 1 to max; + } + } + + optional string version = 1; // The version of the Template to prevent mismatches + repeated Metric metrics = 2; // Each metric includes a name, datatype, and optionally a value + repeated Parameter parameters = 3; + optional string template_ref = 4; // MUST be a reference to a template definition if this is an instance (i.e. the name of the template definition) - MUST be omitted for template definitions + optional bool is_definition = 5; + extensions 6 to max; + } + + message DataSet { + + message DataSetValue { + + oneof value { + uint32 int_value = 1; + uint64 long_value = 2; + float float_value = 3; + double double_value = 4; + bool boolean_value = 5; + string string_value = 6; + DataSetValueExtension extension_value = 7; + } + + message DataSetValueExtension { + extensions 1 to max; + } + } + + message Row { + repeated DataSetValue elements = 1; + extensions 2 to max; // For third party extensions + } + + optional uint64 num_of_columns = 1; + repeated string columns = 2; + repeated uint32 types = 3; + repeated Row rows = 4; + extensions 5 to max; // For third party extensions + } + + message PropertyValue { + + optional uint32 type = 1; + optional bool is_null = 2; + + oneof value { + uint32 int_value = 3; + uint64 long_value = 4; + float float_value = 5; + double double_value = 6; + bool boolean_value = 7; + string string_value = 8; + PropertySet propertyset_value = 9; + PropertySetList propertysets_value = 10; // List of Property Values + PropertyValueExtension extension_value = 11; + } + + message PropertyValueExtension { + extensions 1 to max; + } + } + + message PropertySet { + repeated string keys = 1; // Names of the properties + repeated PropertyValue values = 2; + extensions 3 to max; + } + + message PropertySetList { + repeated PropertySet propertyset = 1; + extensions 2 to max; + } + + message MetaData { + // Bytes specific metadata + optional bool is_multi_part = 1; + + // General metadata + optional string content_type = 2; // Content/Media type + optional uint64 size = 3; // File size, String size, Multi-part size, etc + optional uint64 seq = 4; // Sequence number for multi-part messages + + // File metadata + optional string file_name = 5; // File name + optional string file_type = 6; // File type (i.e. xml, json, txt, cpp, etc) + optional string md5 = 7; // md5 of data + + // Catchalls and future expansion + optional string description = 8; // Could be anything such as json or xml of custom properties + extensions 9 to max; + } + + message Metric { + + optional string name = 1; // Metric name - should only be included on birth + optional uint64 alias = 2; // Metric alias - tied to name on birth and included in all later DATA messages + optional uint64 timestamp = 3; // Timestamp associated with data acquisition time + optional uint32 datatype = 4; // DataType of the metric/tag value + optional bool is_historical = 5; // If this is historical data and should not update real time tag + optional bool is_transient = 6; // Tells consuming clients such as MQTT Engine to not store this as a tag + optional bool is_null = 7; // If this is null - explicitly say so rather than using -1, false, etc for some datatypes. + optional MetaData metadata = 8; // Metadata for the payload + optional PropertySet properties = 9; + + oneof value { + uint32 int_value = 10; + uint64 long_value = 11; + float float_value = 12; + double double_value = 13; + bool boolean_value = 14; + string string_value = 15; + bytes bytes_value = 16; // Bytes, File + DataSet dataset_value = 17; + Template template_value = 18; + MetricValueExtension extension_value = 19; + } + + message MetricValueExtension { + extensions 1 to max; + } + } + + optional uint64 timestamp = 1; // Timestamp at message sending time + repeated Metric metrics = 2; // Repeated forever - no limit in Google Protobufs + optional uint64 seq = 3; // Sequence number + optional string uuid = 4; // UUID to track message type in terms of schema definitions + optional bytes body = 5; // To optionally bypass the whole definition above + extensions 6 to max; // For third party extensions +} \ No newline at end of file diff --git a/05_sparkplugb/src/uns_sparkplugb/__init__.py b/05_sparkplugb/src/uns_sparkplugb/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/05_sparkplugb/src/uns_sparkplugb/generated/__init__.py b/05_sparkplugb/src/uns_sparkplugb/generated/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/05_sparkplugb/src/uns_sparkplugb/generated/sparkplug_b_pb2.py b/05_sparkplugb/src/uns_sparkplugb/generated/sparkplug_b_pb2.py new file mode 100644 index 00000000..fb5cb83c --- /dev/null +++ b/05_sparkplugb/src/uns_sparkplugb/generated/sparkplug_b_pb2.py @@ -0,0 +1,229 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: sparkplug_b.proto +"""Generated protocol buffer code.""" +from google.protobuf.internal import enum_type_wrapper +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x11sparkplug_b.proto\x12\x19org.eclipse.tahu.protobuf\"\xee\x15\n\x07Payload\x12\x11\n\ttimestamp\x18\x01 \x01(\x04\x12:\n\x07metrics\x18\x02 \x03(\x0b\x32).org.eclipse.tahu.protobuf.Payload.Metric\x12\x0b\n\x03seq\x18\x03 \x01(\x04\x12\x0c\n\x04uuid\x18\x04 \x01(\t\x12\x0c\n\x04\x62ody\x18\x05 \x01(\x0c\x1a\xa6\x04\n\x08Template\x12\x0f\n\x07version\x18\x01 \x01(\t\x12:\n\x07metrics\x18\x02 \x03(\x0b\x32).org.eclipse.tahu.protobuf.Payload.Metric\x12I\n\nparameters\x18\x03 \x03(\x0b\x32\x35.org.eclipse.tahu.protobuf.Payload.Template.Parameter\x12\x14\n\x0ctemplate_ref\x18\x04 \x01(\t\x12\x15\n\ris_definition\x18\x05 \x01(\x08\x1a\xca\x02\n\tParameter\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04type\x18\x02 \x01(\r\x12\x13\n\tint_value\x18\x03 \x01(\rH\x00\x12\x14\n\nlong_value\x18\x04 \x01(\x04H\x00\x12\x15\n\x0b\x66loat_value\x18\x05 \x01(\x02H\x00\x12\x16\n\x0c\x64ouble_value\x18\x06 \x01(\x01H\x00\x12\x17\n\rboolean_value\x18\x07 \x01(\x08H\x00\x12\x16\n\x0cstring_value\x18\x08 \x01(\tH\x00\x12h\n\x0f\x65xtension_value\x18\t \x01(\x0b\x32M.org.eclipse.tahu.protobuf.Payload.Template.Parameter.ParameterValueExtensionH\x00\x1a#\n\x17ParameterValueExtension*\x08\x08\x01\x10\x80\x80\x80\x80\x02\x42\x07\n\x05value*\x08\x08\x06\x10\x80\x80\x80\x80\x02\x1a\x97\x04\n\x07\x44\x61taSet\x12\x16\n\x0enum_of_columns\x18\x01 \x01(\x04\x12\x0f\n\x07\x63olumns\x18\x02 \x03(\t\x12\r\n\x05types\x18\x03 \x03(\r\x12<\n\x04rows\x18\x04 \x03(\x0b\x32..org.eclipse.tahu.protobuf.Payload.DataSet.Row\x1a\xaf\x02\n\x0c\x44\x61taSetValue\x12\x13\n\tint_value\x18\x01 \x01(\rH\x00\x12\x14\n\nlong_value\x18\x02 \x01(\x04H\x00\x12\x15\n\x0b\x66loat_value\x18\x03 \x01(\x02H\x00\x12\x16\n\x0c\x64ouble_value\x18\x04 \x01(\x01H\x00\x12\x17\n\rboolean_value\x18\x05 \x01(\x08H\x00\x12\x16\n\x0cstring_value\x18\x06 \x01(\tH\x00\x12h\n\x0f\x65xtension_value\x18\x07 \x01(\x0b\x32M.org.eclipse.tahu.protobuf.Payload.DataSet.DataSetValue.DataSetValueExtensionH\x00\x1a!\n\x15\x44\x61taSetValueExtension*\x08\x08\x01\x10\x80\x80\x80\x80\x02\x42\x07\n\x05value\x1aZ\n\x03Row\x12I\n\x08\x65lements\x18\x01 \x03(\x0b\x32\x37.org.eclipse.tahu.protobuf.Payload.DataSet.DataSetValue*\x08\x08\x02\x10\x80\x80\x80\x80\x02*\x08\x08\x05\x10\x80\x80\x80\x80\x02\x1a\xe9\x03\n\rPropertyValue\x12\x0c\n\x04type\x18\x01 \x01(\r\x12\x0f\n\x07is_null\x18\x02 \x01(\x08\x12\x13\n\tint_value\x18\x03 \x01(\rH\x00\x12\x14\n\nlong_value\x18\x04 \x01(\x04H\x00\x12\x15\n\x0b\x66loat_value\x18\x05 \x01(\x02H\x00\x12\x16\n\x0c\x64ouble_value\x18\x06 \x01(\x01H\x00\x12\x17\n\rboolean_value\x18\x07 \x01(\x08H\x00\x12\x16\n\x0cstring_value\x18\x08 \x01(\tH\x00\x12K\n\x11propertyset_value\x18\t \x01(\x0b\x32..org.eclipse.tahu.protobuf.Payload.PropertySetH\x00\x12P\n\x12propertysets_value\x18\n \x01(\x0b\x32\x32.org.eclipse.tahu.protobuf.Payload.PropertySetListH\x00\x12\x62\n\x0f\x65xtension_value\x18\x0b \x01(\x0b\x32G.org.eclipse.tahu.protobuf.Payload.PropertyValue.PropertyValueExtensionH\x00\x1a\"\n\x16PropertyValueExtension*\x08\x08\x01\x10\x80\x80\x80\x80\x02\x42\x07\n\x05value\x1ag\n\x0bPropertySet\x12\x0c\n\x04keys\x18\x01 \x03(\t\x12@\n\x06values\x18\x02 \x03(\x0b\x32\x30.org.eclipse.tahu.protobuf.Payload.PropertyValue*\x08\x08\x03\x10\x80\x80\x80\x80\x02\x1a`\n\x0fPropertySetList\x12\x43\n\x0bpropertyset\x18\x01 \x03(\x0b\x32..org.eclipse.tahu.protobuf.Payload.PropertySet*\x08\x08\x02\x10\x80\x80\x80\x80\x02\x1a\xa4\x01\n\x08MetaData\x12\x15\n\ris_multi_part\x18\x01 \x01(\x08\x12\x14\n\x0c\x63ontent_type\x18\x02 \x01(\t\x12\x0c\n\x04size\x18\x03 \x01(\x04\x12\x0b\n\x03seq\x18\x04 \x01(\x04\x12\x11\n\tfile_name\x18\x05 \x01(\t\x12\x11\n\tfile_type\x18\x06 \x01(\t\x12\x0b\n\x03md5\x18\x07 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x08 \x01(\t*\x08\x08\t\x10\x80\x80\x80\x80\x02\x1a\xbf\x05\n\x06Metric\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05\x61lias\x18\x02 \x01(\x04\x12\x11\n\ttimestamp\x18\x03 \x01(\x04\x12\x10\n\x08\x64\x61tatype\x18\x04 \x01(\r\x12\x15\n\ris_historical\x18\x05 \x01(\x08\x12\x14\n\x0cis_transient\x18\x06 \x01(\x08\x12\x0f\n\x07is_null\x18\x07 \x01(\x08\x12=\n\x08metadata\x18\x08 \x01(\x0b\x32+.org.eclipse.tahu.protobuf.Payload.MetaData\x12\x42\n\nproperties\x18\t \x01(\x0b\x32..org.eclipse.tahu.protobuf.Payload.PropertySet\x12\x13\n\tint_value\x18\n \x01(\rH\x00\x12\x14\n\nlong_value\x18\x0b \x01(\x04H\x00\x12\x15\n\x0b\x66loat_value\x18\x0c \x01(\x02H\x00\x12\x16\n\x0c\x64ouble_value\x18\r \x01(\x01H\x00\x12\x17\n\rboolean_value\x18\x0e \x01(\x08H\x00\x12\x16\n\x0cstring_value\x18\x0f \x01(\tH\x00\x12\x15\n\x0b\x62ytes_value\x18\x10 \x01(\x0cH\x00\x12\x43\n\rdataset_value\x18\x11 \x01(\x0b\x32*.org.eclipse.tahu.protobuf.Payload.DataSetH\x00\x12\x45\n\x0etemplate_value\x18\x12 \x01(\x0b\x32+.org.eclipse.tahu.protobuf.Payload.TemplateH\x00\x12Y\n\x0f\x65xtension_value\x18\x13 \x01(\x0b\x32>.org.eclipse.tahu.protobuf.Payload.Metric.MetricValueExtensionH\x00\x1a \n\x14MetricValueExtension*\x08\x08\x01\x10\x80\x80\x80\x80\x02\x42\x07\n\x05value*\x08\x08\x06\x10\x80\x80\x80\x80\x02*\xf2\x03\n\x08\x44\x61taType\x12\x0b\n\x07Unknown\x10\x00\x12\x08\n\x04Int8\x10\x01\x12\t\n\x05Int16\x10\x02\x12\t\n\x05Int32\x10\x03\x12\t\n\x05Int64\x10\x04\x12\t\n\x05UInt8\x10\x05\x12\n\n\x06UInt16\x10\x06\x12\n\n\x06UInt32\x10\x07\x12\n\n\x06UInt64\x10\x08\x12\t\n\x05\x46loat\x10\t\x12\n\n\x06\x44ouble\x10\n\x12\x0b\n\x07\x42oolean\x10\x0b\x12\n\n\x06String\x10\x0c\x12\x0c\n\x08\x44\x61teTime\x10\r\x12\x08\n\x04Text\x10\x0e\x12\x08\n\x04UUID\x10\x0f\x12\x0b\n\x07\x44\x61taSet\x10\x10\x12\t\n\x05\x42ytes\x10\x11\x12\x08\n\x04\x46ile\x10\x12\x12\x0c\n\x08Template\x10\x13\x12\x0f\n\x0bPropertySet\x10\x14\x12\x13\n\x0fPropertySetList\x10\x15\x12\r\n\tInt8Array\x10\x16\x12\x0e\n\nInt16Array\x10\x17\x12\x0e\n\nInt32Array\x10\x18\x12\x0e\n\nInt64Array\x10\x19\x12\x0e\n\nUInt8Array\x10\x1a\x12\x0f\n\x0bUInt16Array\x10\x1b\x12\x0f\n\x0bUInt32Array\x10\x1c\x12\x0f\n\x0bUInt64Array\x10\x1d\x12\x0e\n\nFloatArray\x10\x1e\x12\x0f\n\x0b\x44oubleArray\x10\x1f\x12\x10\n\x0c\x42ooleanArray\x10 \x12\x0f\n\x0bStringArray\x10!\x12\x11\n\rDateTimeArray\x10\"B,\n\x19org.eclipse.tahu.protobufB\x0fSparkplugBProto') + +_DATATYPE = DESCRIPTOR.enum_types_by_name['DataType'] +DataType = enum_type_wrapper.EnumTypeWrapper(_DATATYPE) +Unknown = 0 +Int8 = 1 +Int16 = 2 +Int32 = 3 +Int64 = 4 +UInt8 = 5 +UInt16 = 6 +UInt32 = 7 +UInt64 = 8 +Float = 9 +Double = 10 +Boolean = 11 +String = 12 +DateTime = 13 +Text = 14 +UUID = 15 +DataSet = 16 +Bytes = 17 +File = 18 +Template = 19 +PropertySet = 20 +PropertySetList = 21 +Int8Array = 22 +Int16Array = 23 +Int32Array = 24 +Int64Array = 25 +UInt8Array = 26 +UInt16Array = 27 +UInt32Array = 28 +UInt64Array = 29 +FloatArray = 30 +DoubleArray = 31 +BooleanArray = 32 +StringArray = 33 +DateTimeArray = 34 + + +_PAYLOAD = DESCRIPTOR.message_types_by_name['Payload'] +_PAYLOAD_TEMPLATE = _PAYLOAD.nested_types_by_name['Template'] +_PAYLOAD_TEMPLATE_PARAMETER = _PAYLOAD_TEMPLATE.nested_types_by_name['Parameter'] +_PAYLOAD_TEMPLATE_PARAMETER_PARAMETERVALUEEXTENSION = _PAYLOAD_TEMPLATE_PARAMETER.nested_types_by_name['ParameterValueExtension'] +_PAYLOAD_DATASET = _PAYLOAD.nested_types_by_name['DataSet'] +_PAYLOAD_DATASET_DATASETVALUE = _PAYLOAD_DATASET.nested_types_by_name['DataSetValue'] +_PAYLOAD_DATASET_DATASETVALUE_DATASETVALUEEXTENSION = _PAYLOAD_DATASET_DATASETVALUE.nested_types_by_name['DataSetValueExtension'] +_PAYLOAD_DATASET_ROW = _PAYLOAD_DATASET.nested_types_by_name['Row'] +_PAYLOAD_PROPERTYVALUE = _PAYLOAD.nested_types_by_name['PropertyValue'] +_PAYLOAD_PROPERTYVALUE_PROPERTYVALUEEXTENSION = _PAYLOAD_PROPERTYVALUE.nested_types_by_name['PropertyValueExtension'] +_PAYLOAD_PROPERTYSET = _PAYLOAD.nested_types_by_name['PropertySet'] +_PAYLOAD_PROPERTYSETLIST = _PAYLOAD.nested_types_by_name['PropertySetList'] +_PAYLOAD_METADATA = _PAYLOAD.nested_types_by_name['MetaData'] +_PAYLOAD_METRIC = _PAYLOAD.nested_types_by_name['Metric'] +_PAYLOAD_METRIC_METRICVALUEEXTENSION = _PAYLOAD_METRIC.nested_types_by_name['MetricValueExtension'] +Payload = _reflection.GeneratedProtocolMessageType('Payload', (_message.Message,), { + + 'Template' : _reflection.GeneratedProtocolMessageType('Template', (_message.Message,), { + + 'Parameter' : _reflection.GeneratedProtocolMessageType('Parameter', (_message.Message,), { + + 'ParameterValueExtension' : _reflection.GeneratedProtocolMessageType('ParameterValueExtension', (_message.Message,), { + 'DESCRIPTOR' : _PAYLOAD_TEMPLATE_PARAMETER_PARAMETERVALUEEXTENSION, + '__module__' : 'sparkplug_b_pb2' + # @@protoc_insertion_point(class_scope:org.eclipse.tahu.protobuf.Payload.Template.Parameter.ParameterValueExtension) + }) + , + 'DESCRIPTOR' : _PAYLOAD_TEMPLATE_PARAMETER, + '__module__' : 'sparkplug_b_pb2' + # @@protoc_insertion_point(class_scope:org.eclipse.tahu.protobuf.Payload.Template.Parameter) + }) + , + 'DESCRIPTOR' : _PAYLOAD_TEMPLATE, + '__module__' : 'sparkplug_b_pb2' + # @@protoc_insertion_point(class_scope:org.eclipse.tahu.protobuf.Payload.Template) + }) + , + + 'DataSet' : _reflection.GeneratedProtocolMessageType('DataSet', (_message.Message,), { + + 'DataSetValue' : _reflection.GeneratedProtocolMessageType('DataSetValue', (_message.Message,), { + + 'DataSetValueExtension' : _reflection.GeneratedProtocolMessageType('DataSetValueExtension', (_message.Message,), { + 'DESCRIPTOR' : _PAYLOAD_DATASET_DATASETVALUE_DATASETVALUEEXTENSION, + '__module__' : 'sparkplug_b_pb2' + # @@protoc_insertion_point(class_scope:org.eclipse.tahu.protobuf.Payload.DataSet.DataSetValue.DataSetValueExtension) + }) + , + 'DESCRIPTOR' : _PAYLOAD_DATASET_DATASETVALUE, + '__module__' : 'sparkplug_b_pb2' + # @@protoc_insertion_point(class_scope:org.eclipse.tahu.protobuf.Payload.DataSet.DataSetValue) + }) + , + + 'Row' : _reflection.GeneratedProtocolMessageType('Row', (_message.Message,), { + 'DESCRIPTOR' : _PAYLOAD_DATASET_ROW, + '__module__' : 'sparkplug_b_pb2' + # @@protoc_insertion_point(class_scope:org.eclipse.tahu.protobuf.Payload.DataSet.Row) + }) + , + 'DESCRIPTOR' : _PAYLOAD_DATASET, + '__module__' : 'sparkplug_b_pb2' + # @@protoc_insertion_point(class_scope:org.eclipse.tahu.protobuf.Payload.DataSet) + }) + , + + 'PropertyValue' : _reflection.GeneratedProtocolMessageType('PropertyValue', (_message.Message,), { + + 'PropertyValueExtension' : _reflection.GeneratedProtocolMessageType('PropertyValueExtension', (_message.Message,), { + 'DESCRIPTOR' : _PAYLOAD_PROPERTYVALUE_PROPERTYVALUEEXTENSION, + '__module__' : 'sparkplug_b_pb2' + # @@protoc_insertion_point(class_scope:org.eclipse.tahu.protobuf.Payload.PropertyValue.PropertyValueExtension) + }) + , + 'DESCRIPTOR' : _PAYLOAD_PROPERTYVALUE, + '__module__' : 'sparkplug_b_pb2' + # @@protoc_insertion_point(class_scope:org.eclipse.tahu.protobuf.Payload.PropertyValue) + }) + , + + 'PropertySet' : _reflection.GeneratedProtocolMessageType('PropertySet', (_message.Message,), { + 'DESCRIPTOR' : _PAYLOAD_PROPERTYSET, + '__module__' : 'sparkplug_b_pb2' + # @@protoc_insertion_point(class_scope:org.eclipse.tahu.protobuf.Payload.PropertySet) + }) + , + + 'PropertySetList' : _reflection.GeneratedProtocolMessageType('PropertySetList', (_message.Message,), { + 'DESCRIPTOR' : _PAYLOAD_PROPERTYSETLIST, + '__module__' : 'sparkplug_b_pb2' + # @@protoc_insertion_point(class_scope:org.eclipse.tahu.protobuf.Payload.PropertySetList) + }) + , + + 'MetaData' : _reflection.GeneratedProtocolMessageType('MetaData', (_message.Message,), { + 'DESCRIPTOR' : _PAYLOAD_METADATA, + '__module__' : 'sparkplug_b_pb2' + # @@protoc_insertion_point(class_scope:org.eclipse.tahu.protobuf.Payload.MetaData) + }) + , + + 'Metric' : _reflection.GeneratedProtocolMessageType('Metric', (_message.Message,), { + + 'MetricValueExtension' : _reflection.GeneratedProtocolMessageType('MetricValueExtension', (_message.Message,), { + 'DESCRIPTOR' : _PAYLOAD_METRIC_METRICVALUEEXTENSION, + '__module__' : 'sparkplug_b_pb2' + # @@protoc_insertion_point(class_scope:org.eclipse.tahu.protobuf.Payload.Metric.MetricValueExtension) + }) + , + 'DESCRIPTOR' : _PAYLOAD_METRIC, + '__module__' : 'sparkplug_b_pb2' + # @@protoc_insertion_point(class_scope:org.eclipse.tahu.protobuf.Payload.Metric) + }) + , + 'DESCRIPTOR' : _PAYLOAD, + '__module__' : 'sparkplug_b_pb2' + # @@protoc_insertion_point(class_scope:org.eclipse.tahu.protobuf.Payload) + }) +_sym_db.RegisterMessage(Payload) +_sym_db.RegisterMessage(Payload.Template) +_sym_db.RegisterMessage(Payload.Template.Parameter) +_sym_db.RegisterMessage(Payload.Template.Parameter.ParameterValueExtension) +_sym_db.RegisterMessage(Payload.DataSet) +_sym_db.RegisterMessage(Payload.DataSet.DataSetValue) +_sym_db.RegisterMessage(Payload.DataSet.DataSetValue.DataSetValueExtension) +_sym_db.RegisterMessage(Payload.DataSet.Row) +_sym_db.RegisterMessage(Payload.PropertyValue) +_sym_db.RegisterMessage(Payload.PropertyValue.PropertyValueExtension) +_sym_db.RegisterMessage(Payload.PropertySet) +_sym_db.RegisterMessage(Payload.PropertySetList) +_sym_db.RegisterMessage(Payload.MetaData) +_sym_db.RegisterMessage(Payload.Metric) +_sym_db.RegisterMessage(Payload.Metric.MetricValueExtension) + +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + DESCRIPTOR._serialized_options = b'\n\031org.eclipse.tahu.protobufB\017SparkplugBProto' + _DATATYPE._serialized_start=2850 + _DATATYPE._serialized_end=3348 + _PAYLOAD._serialized_start=49 + _PAYLOAD._serialized_end=2847 + _PAYLOAD_TEMPLATE._serialized_start=181 + _PAYLOAD_TEMPLATE._serialized_end=731 + _PAYLOAD_TEMPLATE_PARAMETER._serialized_start=391 + _PAYLOAD_TEMPLATE_PARAMETER._serialized_end=721 + _PAYLOAD_TEMPLATE_PARAMETER_PARAMETERVALUEEXTENSION._serialized_start=677 + _PAYLOAD_TEMPLATE_PARAMETER_PARAMETERVALUEEXTENSION._serialized_end=712 + _PAYLOAD_DATASET._serialized_start=734 + _PAYLOAD_DATASET._serialized_end=1269 + _PAYLOAD_DATASET_DATASETVALUE._serialized_start=864 + _PAYLOAD_DATASET_DATASETVALUE._serialized_end=1167 + _PAYLOAD_DATASET_DATASETVALUE_DATASETVALUEEXTENSION._serialized_start=1125 + _PAYLOAD_DATASET_DATASETVALUE_DATASETVALUEEXTENSION._serialized_end=1158 + _PAYLOAD_DATASET_ROW._serialized_start=1169 + _PAYLOAD_DATASET_ROW._serialized_end=1259 + _PAYLOAD_PROPERTYVALUE._serialized_start=1272 + _PAYLOAD_PROPERTYVALUE._serialized_end=1761 + _PAYLOAD_PROPERTYVALUE_PROPERTYVALUEEXTENSION._serialized_start=1718 + _PAYLOAD_PROPERTYVALUE_PROPERTYVALUEEXTENSION._serialized_end=1752 + _PAYLOAD_PROPERTYSET._serialized_start=1763 + _PAYLOAD_PROPERTYSET._serialized_end=1866 + _PAYLOAD_PROPERTYSETLIST._serialized_start=1868 + _PAYLOAD_PROPERTYSETLIST._serialized_end=1964 + _PAYLOAD_METADATA._serialized_start=1967 + _PAYLOAD_METADATA._serialized_end=2131 + _PAYLOAD_METRIC._serialized_start=2134 + _PAYLOAD_METRIC._serialized_end=2837 + _PAYLOAD_METRIC_METRICVALUEEXTENSION._serialized_start=2796 + _PAYLOAD_METRIC_METRICVALUEEXTENSION._serialized_end=2828 +# @@protoc_insertion_point(module_scope) diff --git a/05_sparkplugb/src/uns_sparkplugb/sparkplugb_enc_config.py b/05_sparkplugb/src/uns_sparkplugb/sparkplugb_enc_config.py new file mode 100644 index 00000000..ebd635a3 --- /dev/null +++ b/05_sparkplugb/src/uns_sparkplugb/sparkplugb_enc_config.py @@ -0,0 +1,13 @@ +from dynaconf import Dynaconf +import os + +current_folder = os.path.dirname(os.path.abspath(__file__)) + +settings = Dynaconf( + envvar_prefix="UNS", + root_path=current_folder, + settings_files=['../../conf/settings.yaml', '../../conf/.secrets.yaml'], +) + +# `envvar_prefix` = export envvars with `export UNS_FOO=bar`. +# `settings_files` = Load these files in the order. diff --git a/05_sparkplugb/src/uns_sparkplugb/spb2unspublisher.py b/05_sparkplugb/src/uns_sparkplugb/spb2unspublisher.py new file mode 100644 index 00000000..1a573b3b --- /dev/null +++ b/05_sparkplugb/src/uns_sparkplugb/spb2unspublisher.py @@ -0,0 +1,328 @@ +import inspect +import json +import logging +import os +import sys +from typing import Any +from paho.mqtt.properties import Properties +from paho.mqtt.packettypes import PacketTypes + +# From http://stackoverflow.com/questions/279237/python-import-a-module-from-a-folder +cmd_subfolder = os.path.realpath( + os.path.abspath( + os.path.join( + os.path.split(inspect.getfile(inspect.currentframe()))[0], '..', + '..', 'src'))) + +uns_mqtt_folder = os.path.realpath( + os.path.abspath( + os.path.join( + os.path.split(inspect.getfile(inspect.currentframe()))[0], '..', + '..', '..', '02_mqtt-cluster', 'src'))) +if cmd_subfolder not in sys.path: + sys.path.insert(0, cmd_subfolder) + +if uns_mqtt_folder not in sys.path: + sys.path.insert(1, uns_mqtt_folder) + +from uns_mqtt.mqtt_listener import Uns_MQTT_ClientWrapper +from uns_sparkplugb.generated import sparkplug_b_pb2 + +LOGGER = logging.getLogger(__name__) + + +class Spb2UNSPublisher: + """ + Class to parse the metric array extracted from the SPB payload and published to the respective UNS topics + Create only one instance of this class per uns_sparkplugb_listener + This is a stateful class and stores the mappings of the alias to topic (if MQTTv5 is used) + This also temporarily stores the mapping of SparkPlugB Metric aliases + """ + SPB_NAME: str = "name" + SPB_ALIAS: str = "alias" + SPB_TIMESTAMP: str = "timestamp" + SPB_DATATYPE: str = "datatype" + SPB_IS_NULL = "is_null" + SPB_IS_HISTORICAL = "is_historical" + + # TODO currently not yet handled correctly. need to figure how to handle this + SPB_METADATA = "metadata" + SPB_PROPERTIES = "properties" + + SPB_DATATYPE_KEYS = { + sparkplug_b_pb2.Unknown: "Unknown", + sparkplug_b_pb2.Int8: "int_value", + sparkplug_b_pb2.Int16: "int_value", + sparkplug_b_pb2.Int32: "int_value", + sparkplug_b_pb2.UInt8: "int_value", + sparkplug_b_pb2.UInt16: "int_value", + sparkplug_b_pb2.UInt32: "int_value", + sparkplug_b_pb2.Int64: "long_value", + sparkplug_b_pb2.UInt64: "long_value", + sparkplug_b_pb2.DateTime: "long_value", + sparkplug_b_pb2.Float: "float_value", + sparkplug_b_pb2.Double: "double_value", + sparkplug_b_pb2.String: "string_value", + sparkplug_b_pb2.Text: "string_value", + sparkplug_b_pb2.UUID: "string_value", + sparkplug_b_pb2.DataSet: "dataset_value", + sparkplug_b_pb2.Bytes: "bytes_value", + sparkplug_b_pb2.File: "bytes_value", + sparkplug_b_pb2.Boolean: "boolean_value", + sparkplug_b_pb2.Template: "template_value", + } + + # stores the message aliases as uses in the SparkPlugB message payload + metric_name_alias_map: dict[int, str] = {} + # Flag indicating this an MQTTv5 connection + isMQTTv5: bool = False + # stores the topic aliases to be used if the protocol is MQTTv5 + topic_alias: dict[str, str] = {} + # the Mqtt client used to publish the messages + mqtt_client: Uns_MQTT_ClientWrapper = None + # maximum topic alias + max_topic_alias: int = 0 + + def __init__(self, mqtt_client: Uns_MQTT_ClientWrapper): + self.mqtt_client = mqtt_client + self.isMQTTv5 = mqtt_client.protocol == Uns_MQTT_ClientWrapper.MQTTv5 + + def clearMetricAlias(self): + """ + Clear all metric aliases stored. + Typically called if a node rebirth / death message is received. + """ + self.metric_name_alias_map.clear() + + def transformSpbAndPublishToUNS(self, + spBPayload, + group_id: str, + message_type: str, + edge_node_id: str, + device_id: str = None) -> dict: + all_uns_messages: dict = {} + match message_type: + case "NBIRTH": # Birth certificate for MQTT EoN nodes. + LOGGER.debug(f"Received message type : {message_type}") + # payload = self.getPayload(spBPayload) + # reset all metric aliases on node birth + self.clearMetricAlias() + all_uns_messages = self.handleSpbMsgs(spBPayload=spBPayload, + group_id=group_id, + message_type=message_type, + edge_node_id=edge_node_id, + device_id=device_id) + + case "NDEATH": # Death certificate for MQTT EoN nodes. + LOGGER.debug(f"Received message type : {message_type}") + # reset all metric aliases on node death + self.clearMetricAlias() + all_uns_messages = self.handleSpbMsgs(spBPayload=spBPayload, + group_id=group_id, + message_type=message_type, + edge_node_id=edge_node_id, + device_id=device_id) + + case "DBIRTH": # Birth certificate for Devices. + LOGGER.debug(f"Received message type : {message_type}") + # at device birth and death there are no metrics published + + case "DDEATH": # Death certificate for Devices. + LOGGER.debug(f"Received message type : {message_type}") + # at device birth and death there are no metrics published + + case "NDATA": # Node data message. + all_uns_messages = self.handleSpbMsgs(spBPayload=spBPayload, + group_id=group_id, + message_type=message_type, + edge_node_id=edge_node_id, + device_id=device_id) + + case "DDATA": # Device data message. + all_uns_messages = self.handleSpbMsgs(spBPayload=spBPayload, + group_id=group_id, + message_type=message_type, + edge_node_id=edge_node_id, + device_id=device_id) + + case "NCMD": # Node command message. + all_uns_messages = self.handleSpbMsgs(spBPayload=spBPayload, + group_id=group_id, + message_type=message_type, + edge_node_id=edge_node_id, + device_id=device_id) + + case "DCMD": # device command message. + all_uns_messages = self.handleSpbMsgs(spBPayload=spBPayload, + group_id=group_id, + message_type=message_type, + edge_node_id=edge_node_id, + device_id=device_id) + + case "STATE": # Critical application state message. + LOGGER.info(f"Received message type : {message_type}") + + case _: + LOGGER.error(f"Unknown message_type received: {message_type}") + raise ValueError(f"Unknown message_type received: {message_type}") + if (len(all_uns_messages) > 0): + self.publishToUNS(all_uns_messages) + + def handleSpbMsgs(self, + spBPayload, + group_id: str, + message_type: str, + edge_node_id: str, + device_id: str = None) -> dict: + """ + Parse the SPBPayload for message types + """ + # convert spb payload and extract metrics array + metrics_list: list = self.getMetricsListFromPayload(spBPayload) + + # collate all metrics to the same topic and send them as one payload + all_uns_messages: dict = {} + spbContext = self.getSpbContext(group_id, message_type, + edge_node_id, device_id) + for metric in metrics_list: + name = self.getMetricName(metric) + name_list = name.rsplit("/", 1) + uns_topic = name_list[0] + tag_name = name_list[1] + + metric_timestamp: float = float(getattr(metric, Spb2UNSPublisher.SPB_TIMESTAMP)) + datatype: int = getattr(metric, Spb2UNSPublisher.SPB_DATATYPE) + + isHistorical: bool = getattr(metric, Spb2UNSPublisher.SPB_IS_HISTORICAL, False) + metric_value = None + if (not getattr(metric, Spb2UNSPublisher.SPB_IS_NULL)): + metric_value = getattr(metric, Spb2UNSPublisher.SPB_DATATYPE_KEYS.get(datatype)) + + uns_message: dict[str, Any] = Spb2UNSPublisher.extractUNSMessageForTopic( + parsed_message=all_uns_messages.get(uns_topic), + tag_name=tag_name, + metric_value=metric_value, + metric_timestamp=metric_timestamp, + isHistorical=isHistorical, + spbContext=spbContext) + + all_uns_messages[uns_topic] = uns_message + # Publish all messages to the respective UNS topics + + return all_uns_messages + + def getMetricName(self, metric): + """ + Extract metric name from metric payload + Encapsulate metric name alias handling + """ + name: str = getattr(metric, Spb2UNSPublisher.SPB_NAME, None) + try: + metric_alias: int = int(getattr(metric, Spb2UNSPublisher.SPB_ALIAS)) + except (AttributeError, TypeError, ValueError): + metric_alias = None + + if (name is None or name == ""): + if (metric_alias is not None): + name = self.metric_name_alias_map.get(metric_alias) + if (name is None or name == ""): + LOGGER.error( + f"Skipping as metric Name is null and alias not yet provided: {metric}" + ) + elif (metric_alias is not None): + # if metric_alias was provided then store it in the map + self.metric_name_alias_map[metric_alias] = name + return name + + @staticmethod + def getSpbContext(group_id: str, message_type: str, edge_node_id: str, + device_id: str) -> dict[str, str]: + """ + returns a dict of key value pair of the SPB parsed topic + """ + spbContext = { + "spBv1.0_group_id": group_id, + "spBv1.0_message_type": message_type, + "spBv1.0_edge_node_id": edge_node_id + } + if (device_id is not None): + spbContext["spBv1.0_device_id"] = device_id + return spbContext + + @staticmethod + def getMetricsListFromPayload(payload) -> list: + """ + Extracts metrics list from the SparkplugB payload + """ + inboundPayload = sparkplug_b_pb2.Payload() + inboundPayload.ParseFromString(payload) + return inboundPayload.metrics + + @staticmethod + def getPayload(payload) -> dict: + """ + Converts SparkplugB payload to a dict + """ + inboundPayload = sparkplug_b_pb2.Payload() + inboundPayload.ParseFromString(payload) + return inboundPayload + + @staticmethod + def extractUNSMessageForTopic( + parsed_message: dict, + tag_name: str, + metric_value: Any, + metric_timestamp: float, + isHistorical: bool = False, + spbContext: dict[str, str] = None) -> dict[str, Any]: + """ + Returns a dictionary where the key is the target topic name and value is the message payload in dict format + """ + # check if there were any tags already parsed for this uns topic + if (parsed_message is None): + parsed_message = { + tag_name: (metric_value, metric_timestamp, isHistorical) + } + parsed_message[Spb2UNSPublisher.SPB_TIMESTAMP] = metric_timestamp + # enrich the message to add SpB related information + if (spbContext is not None): + parsed_message.update(spbContext) + else: + # check if there were there was any already parsed metric for this tag and UNS topic + old_metric_tuple_list = parsed_message.get(tag_name) + old_metric_timestamp = parsed_message.get(Spb2UNSPublisher.SPB_TIMESTAMP) + if (not isinstance(old_metric_tuple_list, list)): + # replace current metric value with array of values if it is a singe value + old_metric_tuple_list = [old_metric_tuple_list] + + # If the value for the metric name already exists then we will have a list of tuples + if (old_metric_timestamp > metric_timestamp): + # if the metric in the uns_message is newer than the metric received add the new value at the end + # TODO currently for simplicity have not yet ordered all old values, just the newest is at 0 + # TODO storing value, timestamp and isHistorical flag as a tuple. + # If needed might need to convert this into a dict to have attribute keys for identification + old_metric_tuple_list.append( + (metric_value, metric_timestamp, isHistorical)) + else: + # if the metric in the uns_message is older than the metric received + old_metric_tuple_list.insert( + 0, (metric_value, metric_timestamp, isHistorical)) + parsed_message[Spb2UNSPublisher.SPB_TIMESTAMP] = metric_timestamp + parsed_message[tag_name] = old_metric_tuple_list + # end of inner if & else + # end of outer if + return parsed_message + + def publishToUNS(self, all_uns_messages: dict): + if (self.mqtt_client.is_connected()): + for topic in all_uns_messages: + message = all_uns_messages.get(topic) + publish_properties = None + if (self.isMQTTv5): + publish_properties = Properties(PacketTypes.PUBLISH) + + self.mqtt_client.publish(topic=topic, payload=json.dumps(message), qos=self.mqtt_client.qos, + retain=True, properties=publish_properties) + else: + LOGGER.error(f"MQTT Client is not connected. Cannot publish to UNS. {self.mqtt_client}") + raise ConnectionError(f"{self.mqtt_client}") diff --git a/05_sparkplugb/src/uns_sparkplugb/uns_sparkplug_b_gen.py b/05_sparkplugb/src/uns_sparkplugb/uns_sparkplug_b_gen.py new file mode 100644 index 00000000..bbb14ba7 --- /dev/null +++ b/05_sparkplugb/src/uns_sparkplugb/uns_sparkplug_b_gen.py @@ -0,0 +1,310 @@ +# /** +# * Helper class to parse & create SparkplugB messages +# * @see Tahu Project{https://github.com/eclipse/tahu/blob/master/python/core/sparkplug_b.py} +# */ +import logging +from uns_sparkplugb.generated import sparkplug_b_pb2 +import time + +LOGGER = logging.getLogger(__name__) + + +class Spb_Message_Generator: + # sequence number for messages + seqNum: int = 0 + # birth/death sequence number + bdSeq: int = 0 + + # map of metric names to alias. While adding metrics, if an alias exists for that name it will be used instead + aliasMap: dict[str, str] = {} + def __init__(self) -> None: + self.aliasMap = dict() + + def getSeqNum(self): + """ + Helper method for getting the next sequence number + """ + retVal = self.seqNum + LOGGER.debug(f"Sequence Number: {str(retVal)}") + self.seqNum += 1 + if self.seqNum == 256: + self.seqNum = 0 + return retVal + + def getBdSeqNum(self): + """ + Helper method for getting the next birth/death sequence number + """ + retVal = self.bdSeq + LOGGER.debug(f"Birth/Death Sequence Number: {str(retVal)}") + self.bdSeq += 1 + if self.bdSeq == 256: + self.bdSeq = 0 + return retVal + + def getNodeDeathPayload(self, payload=None): + """ + Helper to get the Death Node Payload + Always request this before requesting the Node Birth Payload + """ + if (payload is None): + payload = sparkplug_b_pb2.Payload() + self.addMetric(payload, "bdSeq", None, sparkplug_b_pb2.Int64, + self.getBdSeqNum()) + return payload + + def getNodeBirthPayload(self, payload=None): + """ + Helper to get the Node Birth Payload + Always request this after requesting the Node Death Payload + """ + self.seqNum = 0 + if (payload is None): + payload = sparkplug_b_pb2.Payload() + payload.timestamp = int(round(time.time() * 1000)) + payload.seq = self.getSeqNum() + self.addMetric(payload, "bdSeq", None, sparkplug_b_pb2.Int64, + --self.bdSeq, payload.timestamp) + return payload + + def getDeviceBirthPayload(self, payload=None): + """ + Get the DBIRTH payload + """ + if (payload is None): + payload = sparkplug_b_pb2.Payload() + payload.timestamp = int(round(time.time() * 1000)) + payload.seq = self.getSeqNum() + return payload + + ###################################################################### + # Get a DDATA payload + ###################################################################### + def getDeviceDataPayload(self, payload=None): + """ + @TODO review this + """ + return self.getDeviceBirthPayload(payload) + + ###################################################################### + # Get a NDATA payload + ###################################################################### + def getNodeDataPayload(self, payload=None): + """ + @TODO review this + """ + return self.getNodeBirthPayload(payload) + + ###################################################################### + # Refactored common code of obtaining metrics and initializing + # common attributes + ###################################################################### + def getMetricWrapper(self, + payload, + name, + alias, + timestamp=int(round(time.time() * 1000))): + metric = payload.metrics.add() + if alias is not None: + metric.alias = alias + + if name is not None: + metric.name = name + + if self.aliasMap.get(name) is None: + self.aliasMap[name] = alias + elif self.aliasMap.get(name) == alias: + metric.name = None + else: + raise ValueError(f"Alias:{alias} provided for Name:{name} not matching to previously provided alias:{self.aliasMap.get(name)} ") + metric.timestamp = timestamp + return metric + + ###################################################################### + # Helper method for adding dataset metrics to a payload + ###################################################################### + def initDatasetMetric(self, payload, name, alias, columns, types): + metric = self.getMetricWrapper(payload, name, alias) + + metric.datatype = sparkplug_b_pb2.DataSet + # Set up the dataset + metric.dataset_value.num_of_columns = len(types) + metric.dataset_value.columns.extend(columns) + metric.dataset_value.types.extend(types) + return metric.dataset_value + + ###################################################################### + # Helper method for adding dataset metrics to a payload + ###################################################################### + def initTemplateMetric(self, payload, name, alias, templateRef): + metric = self.getMetricWrapper(payload, name, alias) + metric.datatype = sparkplug_b_pb2.Template + + # Set up the template + if templateRef is not None: + metric.template_value.template_ref = templateRef + metric.template_value.is_definition = False + else: + metric.template_value.is_definition = True + + return metric.template_value + + ###################################################################### + # + ###################################################################### + def addMetric(self, + payload, + name, + alias, + type, + value=None, + timestamp=int(round(time.time() * 1000))): + """ + Helper method for adding metrics to a container which can be a payload or a template. + Parameters + ---------- + payload: the Payload object + name: Name of the metric.May be hierarchical to build out proper folder structures for applications + consuming the metric values + alias: Alias. unsigned 64-bit integer representing an optional alias for a Sparkplug B payload + type, + value, + timestamp + """ + metric = self.getMetricWrapper(payload, name, alias, timestamp) + if value is None: + metric.is_null = True + metric.datatype = type + + match type: + case sparkplug_b_pb2.Int8: setIntValueInMetric(value, metric, 8), + case sparkplug_b_pb2.Int16: setIntValueInMetric(value, metric, 16), + case sparkplug_b_pb2.Int32: setIntValueInMetric(value, metric, 32), + case sparkplug_b_pb2.Int64: setLongValueInMetric(value, metric, 64), + case sparkplug_b_pb2.UInt8: setIntValueInMetric(value, metric, 0), + case sparkplug_b_pb2.UInt16: setIntValueInMetric(value, metric, 0), + case sparkplug_b_pb2.UInt32: setIntValueInMetric(value, metric, 0), + case sparkplug_b_pb2.UInt64: setLongValueInMetric(value, metric, 0), + case sparkplug_b_pb2.Float: setFloatValueInMetric(value, metric), + case sparkplug_b_pb2.Double: setDoubleValueInMetric(value, metric), + case sparkplug_b_pb2.Boolean: setBooleanValueInMetric(value, metric), + case sparkplug_b_pb2.String: setStringValueInMetric(value, metric), + case sparkplug_b_pb2.DateTime: setLongValueInMetric(value, metric, 0), + case sparkplug_b_pb2.Text: setStringValueInMetric(value, metric), + case sparkplug_b_pb2.UUID: setStringValueInMetric(value, metric), + case sparkplug_b_pb2.DataSet: + raise ValueError(f"MetricType:{sparkplug_b_pb2.DataSet} Not supported by #addMetric(). Use #initDatasetMetric()") + case sparkplug_b_pb2.Bytes: setBytesValueInMetric(value, metric), + case sparkplug_b_pb2.File: setBytesValueInMetric(value, metric), + case sparkplug_b_pb2.Template: setTemplatesValueInMetric(value, metric) + case _: unknownValueTypeInMetric(value, metric) + + # Return the metric + return metric + + ###################################################################### + # Helper method for adding metrics to a container which can be a + # payload or a template + ###################################################################### + def addHistoricalMetric(self, container, name, alias, type, value): + metric = self.addMetric(container, name, alias, type, value) + metric.is_historical = True + # Return the metric + return metric + + ##################################################################### + # Helper method for adding metrics to a container which can be a + # payload or a template + ###################################################################### + def addNullMetric(self, container, name, alias, type): + metric = self.addMetric(container, name, alias, type) + metric.is_null = True + return metric + + +###################################################################### +# +###################################################################### +@staticmethod +def setIntValueInMetric(value, metric, factor): + if value is not None and value < 0: + value = value + (0 if factor == 0 else 2**factor) + metric.int_value = value + return value + + +###################################################################### +# +###################################################################### +@staticmethod +def setLongValueInMetric(value, metric, factor=0): + if value is not None and value < 0: + value = value + (2**factor if factor != 0 else 0) + metric.long_value = value + return value + + +###################################################################### +# +###################################################################### +@staticmethod +def setFloatValueInMetric(value, metric): + metric.float_value = value + return value + + +###################################################################### +# +###################################################################### +@staticmethod +def setDoubleValueInMetric(value, metric): + metric.double_value = value + return value + + +###################################################################### +# +###################################################################### +@staticmethod +def setStringValueInMetric(value, metric): + metric.string_value = value + return value + + +###################################################################### +# +###################################################################### +@staticmethod +def setBytesValueInMetric(value, metric): + metric.bytes_value = value + return value + + +###################################################################### +# +###################################################################### +@staticmethod +def setTemplatesValueInMetric(value, metric): + metric.template_value = value + return value + + +###################################################################### +# +###################################################################### +@staticmethod +def setBooleanValueInMetric(value: bool, metric): + metric.boolean_value = value + return value + + +###################################################################### +# +###################################################################### +@staticmethod +def unknownValueTypeInMetric(value, metric): + metric.datatype = None + LOGGER.error( + f"Invalid type: {type}.\n Value:{value} not added to {metric}", + stack_info=True, + exc_info=True) diff --git a/05_sparkplugb/src/uns_sparkplugb/uns_sparkplugb_listner.py b/05_sparkplugb/src/uns_sparkplugb/uns_sparkplugb_listner.py new file mode 100644 index 00000000..966d7121 --- /dev/null +++ b/05_sparkplugb/src/uns_sparkplugb/uns_sparkplugb_listner.py @@ -0,0 +1,145 @@ +import inspect +import random +import logging +import time +import os +import sys +from sparkplugb_enc_config import settings +# From http://stackoverflow.com/questions/279237/python-import-a-module-from-a-folder +cmd_subfolder = os.path.realpath( + os.path.abspath( + os.path.join( + os.path.split(inspect.getfile(inspect.currentframe()))[0], '..', + '..', '..', '02_mqtt-cluster', 'src'))) +if cmd_subfolder not in sys.path: + sys.path.insert(0, cmd_subfolder) + sys.path.insert( + 0, + os.path.abspath( + os.path.join(cmd_subfolder, "..", "..", "05_sparkplugb", "src"))) + +from uns_mqtt.mqtt_listener import Uns_MQTT_ClientWrapper +from spb2unspublisher import Spb2UNSPublisher + +LOGGER = logging.getLogger(__name__) + + +# listens to SparkplugB name space for messages and publishes to ISA-95 +# https://www.hivemq.com/solutions/manufacturing/smart-manufacturing-using-isa95-mqtt-sparkplug-and-uns/ +class Uns_SparkPlugB_Mapper: + + def __init__(self): + self.uns_client: Uns_MQTT_ClientWrapper = None + self.load_mqtt_configs() + self.load_sparkplugb_configs() + self.uns_client: Uns_MQTT_ClientWrapper = Uns_MQTT_ClientWrapper( + client_id=self.client_id, + clean_session=self.clean_session, + userdata=None, + protocol=self.mqtt_mqtt_version_code, + transport=self.mqtt_transport, + reconnect_on_failure=self.reconnect_on_failure) + + self.uns_client.on_message = self.on_message + self.uns_client.on_disconnect = self.on_disconnect + + self.spg2unPub: Spb2UNSPublisher = Spb2UNSPublisher(self.uns_client) + self.uns_client.run(host=self.mqtt_host, + port=self.mqtt_port, + username=self.mqtt_username, + password=self.mqtt_password, + tls=self.mqtt_tls, + keepalive=self.mqtt_keepalive, + topic=self.topic, + qos=self.mqtt_qos) + + def load_mqtt_configs(self): + # generate client ID with pub prefix randomly + self.client_id = f'uns_sparkplugb_listener-{time.time()}-{random.randint(0, 1000)}' + + self.mqtt_transport: str = settings.get("mqtt.transport", "tcp") + self.mqtt_mqtt_version_code: int = settings.get( + "mqtt.version", Uns_MQTT_ClientWrapper.MQTTv5) + self.mqtt_qos: int = settings.get("mqtt.qos", 2) + self.reconnect_on_failure: bool = settings.get( + "mqtt.reconnect_on_failure", True) + self.clean_session: bool = settings.get("mqtt.clean_session", None) + + self.mqtt_host: str = settings.mqtt["host"] + self.mqtt_port: int = settings.get("mqtt.port", 1883) + self.mqtt_username: str = settings.mqtt["username"] + self.mqtt_password: str = settings.mqtt["password"] + self.mqtt_tls: dict = settings.get("mqtt.tls", None) + self.topic: str = settings.get("mqtt.topic", "spBv1.0/#") + self.mqtt_keepalive: int = settings.get("mqtt.keep_alive", 60) + self.mqtt_ignored_attributes: dict = settings.get( + "mqtt.ignored_attributes", None) + self.mqtt_timestamp_key = settings.get("mqtt.timestamp_attribute", + "timestamp") + if (self.mqtt_host is None): + raise ValueError( + "MQTT Host not provided. Update key 'mqtt.host' in '../../conf/settings.yaml'" + ) + + def load_sparkplugb_configs(self): + """ + Load the SparkplugB configurations + """ + # Currently no configurations + + def on_message(self, client, userdata, msg): + """ + Callback function executed every time a message is received by the subscriber + """ + LOGGER.debug("{" + f"Client: {client}," + f"Userdata: {userdata}," + f"Message: {msg}," + "}") + try: + topics: list[str] = msg.topic.split('/') + # sPB topic structure spBv1.0////<[device_id]> + # device_id is optional. all others are mandatory + # if (len(topics) == 4 or len(topics) == 5): + if (len(topics) >= 4): + group_id = topics[1] + message_type = topics[2] + edge_node_id = topics[3] + device_id = None + if (len(topics) == 5): + device_id = topics[4] + self.spg2unPub.transformSpbAndPublishToUNS( + msg.payload, group_id, message_type, edge_node_id, + device_id) + else: + raise ValueError( + f"Unknown SparkplugB topic received: {msg.topic}") + + except Exception as ex: + LOGGER.error("Error parsing SparkplugB message payload: %s", + str(ex), + stack_info=True, + exc_info=True) + raise ex + + def on_disconnect(self, client, userdata, rc, properties=None): + # Cleanup when the MQTT broker gets disconnected + LOGGER.debug("SparkplugB listener got disconnected") + if (rc != 0): + LOGGER.error("Unexpected disconnection.:%s", + str(rc), + stack_info=True, + exc_info=True) + + +def main(): + try: + uns_spb_mapper = Uns_SparkPlugB_Mapper() + uns_spb_mapper.uns_client.loop_forever() + finally: + if (uns_spb_mapper is not None): + uns_spb_mapper.uns_client.disconnect() + + +if __name__ == '__main__': + main() diff --git a/05_sparkplugb/test/__init__.py b/05_sparkplugb/test/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/05_sparkplugb/test/test_sparkplugb_enc_config.py b/05_sparkplugb/test/test_sparkplugb_enc_config.py new file mode 100644 index 00000000..719b0822 --- /dev/null +++ b/05_sparkplugb/test/test_sparkplugb_enc_config.py @@ -0,0 +1,111 @@ +import inspect +import os +import re +import socket +import sys +import pytest + +# From http://stackoverflow.com/questions/279237/python-import-a-module-from-a-folder +cmd_subfolder = os.path.realpath( + os.path.abspath( + os.path.join( + os.path.split(inspect.getfile(inspect.currentframe()))[0], '..', + 'src'))) +if cmd_subfolder not in sys.path: + sys.path.insert(0, cmd_subfolder) + sys.path.insert(1, os.path.join(cmd_subfolder, "uns_sparkplugb")) +from sparkplugb_enc_config import settings + +is_configs_provided: bool = ( + os.path.exists(os.path.join(cmd_subfolder, "../conf/.secrets.yaml")) + and os.path.exists(os.path.join(cmd_subfolder, "../conf/settings.yaml")) + or (bool(os.getenv("UNS_mqtt.host")))) + + +@pytest.mark.xfail(not is_configs_provided, + reason="Configurations have not been provided") +def test_mqtt_config(): + # run these tests only if both configuration files exists or mandatory environment vars are set + mqtt_transport: str = settings.get("mqtt.transport") + assert mqtt_transport in ( + None, "tcp", + "ws"), f"Invalid value for key 'mqtt.transport':{mqtt_transport}" + + mqtt_version: int = settings.get("mqtt.version") + assert mqtt_version in ( + None, 3, 4, 5), f"Invalid value for key 'mqtt.version':{mqtt_version}" + + mqtt_qos: int = settings.get("mqtt.qos") + assert mqtt_qos in (None, 0, 1, + 2), f"Invalid value for key 'mqtt.qos':{mqtt_qos}" + + reconnect_on_failure: bool = settings.get("mqtt.reconnect_on_failure") + assert reconnect_on_failure in ( + None, True, False + ), f"Invalid value for key 'mqtt.reconnect_on_failure'{reconnect_on_failure}" + + clean_session: bool = settings.get("mqtt.clean_session") + assert clean_session in ( + True, False, + None), f"Invalid value for key 'mqtt.clean_session'{clean_session}" + + host: str = settings.mqtt["host"] + assert host is not None, f"Invalid value for key 'mqtt.host'{host}" + + port: int = settings.get("mqtt.port", 1883) + assert type( + port + ) is int or port is None, f"Invalid value for key 'mqtt.port':{str(port)}" + assert type( + port + ) is int and port >= 1024 and port <= 49151, f"'mqtt.port':{str(port)} must be between 1024 to 49151" + + username = settings.mqtt["username"] + password = settings.mqtt["password"] + assert (username is None and password is None) or ( + type(username) is str and type(password) is str + ), "Either both username & password need to be specified or neither" + + tls: dict = settings.get("mqtt.tls", None) + assert (tls is None) or ( + type(tls) is dict and not bool(tls) and tls.get("ca_certs") is not None + ), "Check the configuration provided for tls connection to the broker. the property ca_certs is missing" + + assert (tls is None) or (os.path.isfile(tls.get( + "ca_certs"))), f"Unable to find certificate at: {tls.get('ca_certs')}" + + topic: str = settings.get("mqtt.topic", "#") + REGEX_TO_MATCH_TOPIC = r"^(\+|\#|.+/\+|[^#]+#|.*/\+/.*)$" + assert bool( + re.fullmatch(REGEX_TO_MATCH_TOPIC, topic) + ), f"configuration 'mqtt.topic':{topic} is not a valid MQTT topic" + + keep_alive: float = settings.get("mqtt.keep_alive", 60) + assert (keep_alive is None) or ( + keep_alive > + 0), f"'mqtt.keep_alive'{keep_alive} must be a positive number" + + ignored_attributes: dict = settings.get("mqtt.ignored_attributes") + assert (ignored_attributes is None) or ( + isinstance(ignored_attributes, dict) + ), f"Configuration 'mqtt.ignored_attributes':{ignored_attributes} is not a valid dict" + + timestamp_attribute: str = settings.get("mqtt.timestamp_attribute", + "timestamp") + # Should be a valid JSON attribute + assert (timestamp_attribute is None) or ( + len(timestamp_attribute) > 0 + ), f"Configuration 'mqtt.timestamp_attribute':{timestamp_attribute} is not a valid JSON key" + + +@pytest.mark.integrationtest +@pytest.mark.xfail( + not is_configs_provided, + reason="Configurations absent, or these are not integration tests") +def test_connectivity_to_mqtt(): + host: str = settings.mqtt["host"] + port: int = settings.get("mqtt.port", 1883) + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + assert sock.connect_ex( + (host, + port)) == 0, f"Host: {host} is not reachable at port:{str(port)}" diff --git a/05_sparkplugb/test/test_spb2unspublisher.py b/05_sparkplugb/test/test_spb2unspublisher.py new file mode 100644 index 00000000..8cb0d602 --- /dev/null +++ b/05_sparkplugb/test/test_spb2unspublisher.py @@ -0,0 +1,445 @@ +import inspect +import os +import sys +import time +from types import SimpleNamespace +import pytest + +cmd_subfolder = os.path.realpath( + os.path.abspath( + os.path.join( + os.path.split(inspect.getfile(inspect.currentframe()))[0], '..', + 'src'))) +uns_mqtt_folder = os.path.realpath( + os.path.abspath( + os.path.join(cmd_subfolder, '..', '..', '02_mqtt-cluster', 'src'))) + +if cmd_subfolder not in sys.path: + sys.path.insert(0, cmd_subfolder) + +if uns_mqtt_folder not in sys.path: + sys.path.insert(2, uns_mqtt_folder) + +from uns_mqtt.mqtt_listener import Uns_MQTT_ClientWrapper +from uns_sparkplugb import uns_sparkplug_b_gen +from uns_sparkplugb.spb2unspublisher import Spb2UNSPublisher +from uns_sparkplugb.generated import sparkplug_b_pb2 + + +@pytest.mark.parametrize("clean_session", [(True), (False)]) +@pytest.mark.parametrize("reconnect_on_failure", [(True)]) +@pytest.mark.parametrize("protocol", [(Uns_MQTT_ClientWrapper.MQTTv5), + (Uns_MQTT_ClientWrapper.MQTTv311)]) +@pytest.mark.parametrize("qos", [(1), (2)]) +@pytest.mark.parametrize("transport,host, port, tls", + [("tcp", "broker.emqx.io", 1883, None)]) +def test_Spb2UNSPublisher_init(clean_session, protocol, transport, host, port, + tls, qos, reconnect_on_failure): + """ + See Spb2UNSPublisher#init() + """ + uns_client: Uns_MQTT_ClientWrapper = Uns_MQTT_ClientWrapper( + client_id=f"spBv1.0_mapper_tester_{time.time()}", + clean_session=clean_session, + userdata=None, + protocol=protocol, + transport=transport, + reconnect_on_failure=reconnect_on_failure) + # Connection not made to broker + spg2unPub = Spb2UNSPublisher(uns_client) + if protocol == Uns_MQTT_ClientWrapper.MQTTv5: + assert ( + spg2unPub.isMQTTv5 is True + ), f"Spb2UNSPublisher#isMQTTv5 should have been True but was {spg2unPub.isMQTTv5}" + else: + assert ( + spg2unPub.isMQTTv5 is False + ), f"Spb2UNSPublisher#isMQTTv5 should have been False but was {spg2unPub.isMQTTv5}" + + assert spg2unPub.mqtt_client == uns_client, "Spb2UNSPublisher#mqtt_client should have been correctly initialized" + assert len(spg2unPub.metric_name_alias_map + ) == 0, "Spb2UNSPublisher#message_alias_map should be empty" + assert len(spg2unPub.topic_alias + ) == 0, "Spb2UNSPublisher#topic_alias should be empty" + + +@pytest.mark.parametrize("clean_session", [(True), (False)]) +@pytest.mark.parametrize("reconnect_on_failure", [(True)]) +@pytest.mark.parametrize("protocol", [(Uns_MQTT_ClientWrapper.MQTTv5), + (Uns_MQTT_ClientWrapper.MQTTv311)]) +@pytest.mark.parametrize("qos", [(1), (2)]) +@pytest.mark.parametrize("transport,host, port, tls", + [("tcp", "broker.emqx.io", 1883, None)]) +def test_Spb2UNSPublisher_clearMetricAlias(clean_session, protocol, transport, + host, port, tls, qos, + reconnect_on_failure): + """ + See Spb2UNSPublisher#clearMetricAlias() + """ + uns_client: Uns_MQTT_ClientWrapper = Uns_MQTT_ClientWrapper( + client_id=f"spBv1.0_mapper_tester_{time.time()}", + clean_session=clean_session, + userdata=None, + protocol=protocol, + transport=transport, + reconnect_on_failure=reconnect_on_failure) + # Connection not made to broker + spg2unPub = Spb2UNSPublisher(uns_client) + spg2unPub.metric_name_alias_map[1] = "value1" + spg2unPub.metric_name_alias_map[2] = "value2" + spg2unPub.metric_name_alias_map[3] = "value3" + assert len(spg2unPub.metric_name_alias_map + ) == 3, "Spb2UNSPublisher#message_alias_map should not be empty" + spg2unPub.clearMetricAlias() + assert len(spg2unPub.metric_name_alias_map + ) == 0, "Spb2UNSPublisher#message_alias_map should be empty" + + +@pytest.mark.parametrize( + "metric,name", + [(SimpleNamespace(**{Spb2UNSPublisher.SPB_NAME: "metric_name"}), + "metric_name"), + (SimpleNamespace( + **{ + Spb2UNSPublisher.SPB_NAME: "metric_name1", + Spb2UNSPublisher.SPB_ALIAS: 1, + Spb2UNSPublisher.SPB_DATATYPE: "Int8" + }), "metric_name1")]) +def testGetMetricName(metric, name: str): + """ + See Spb2UNSPublisher#getMetricName() + """ + uns_client: Uns_MQTT_ClientWrapper = Uns_MQTT_ClientWrapper( + client_id=f"spBv1.0_mapper_tester_{time.time()}", + clean_session=True, + userdata=None, + protocol=Uns_MQTT_ClientWrapper.MQTTv5, + transport="tcp", + reconnect_on_failure=True) + # Connection not made to broker + spg2unPub = Spb2UNSPublisher(uns_client) + assert spg2unPub.getMetricName( + metric + ) == name, f"Incorrect Name: {name} retrieved from metric: {metric}." + + +@pytest.mark.parametrize("metric", + [(SimpleNamespace(**{"no_name": "bad key"})), + (SimpleNamespace(**{})), (None)]) +def testNegativeGetMetricName(metric): + """ + See Spb2UNSPublisher#getMetricName() + """ + uns_client: Uns_MQTT_ClientWrapper = Uns_MQTT_ClientWrapper( + client_id=f"spBv1.0_mapper_tester_{time.time()}", + clean_session=True, + userdata=None, + protocol=Uns_MQTT_ClientWrapper.MQTTv5, + transport="tcp", + reconnect_on_failure=True) + # Connection not made to broker + spg2unPub = Spb2UNSPublisher(uns_client) + assert spg2unPub.getMetricName( + metric) is None, f"Metric name should be none for metric:{metric}" + + +@pytest.mark.parametrize("metric,name, alias", [(SimpleNamespace( + **{ + Spb2UNSPublisher.SPB_NAME: "metric_name1", + Spb2UNSPublisher.SPB_ALIAS: 1, + Spb2UNSPublisher.SPB_DATATYPE: "Int8" + }), "metric_name1", 1)]) +def testGetMetricName_from_Alias(metric, name: str, alias: int): + """ + See Spb2UNSPublisher#getMetricName() + """ + uns_client: Uns_MQTT_ClientWrapper = Uns_MQTT_ClientWrapper( + client_id=f"spBv1.0_mapper_tester_{time.time()}", + clean_session=True, + userdata=None, + protocol=Uns_MQTT_ClientWrapper.MQTTv5, + transport="tcp", + reconnect_on_failure=True) + # Connection not made to broker + spg2unPub = Spb2UNSPublisher(uns_client) + + assert spg2unPub.getMetricName( + metric + ) == name, f"Incorrect Name: {name} retrieved from metric: {metric}." + + assert spg2unPub.metric_name_alias_map[alias] == name, ( + f"Metric AliasMap was not filled {spg2unPub.metric_name_alias_map}" + f"with alias: {alias} and name: {name}.") + # remove the name from the metric and check again. The alias map should ensure that the name is returned + delattr(metric, Spb2UNSPublisher.SPB_NAME) + + assert spg2unPub.getMetricName( + metric + ) == name, f"Incorrect Name: {name} retrieved from metric: {metric}." + assert spg2unPub.metric_name_alias_map[alias] == name, ( + f"Metric AliasMap was not filled {spg2unPub.metric_name_alias_map}" + f"with alias: {alias} and name: {name}.") + + +@pytest.mark.parametrize( + "group_id,message_type, edge_node_id, device_id, expected_ctx", + [("grp1", "DDATA", "node1", "dev1", { + "spBv1.0_group_id": "grp1", + "spBv1.0_message_type": "DDATA", + "spBv1.0_edge_node_id": "node1", + "spBv1.0_device_id": "dev1" + }), + ("grp1", "DDATA", "node1", None, { + "spBv1.0_group_id": "grp1", + "spBv1.0_message_type": "DDATA", + "spBv1.0_edge_node_id": "node1" + })]) +def testGetSpbContext(group_id: str, message_type: str, edge_node_id: str, + device_id: str, expected_ctx: dict[str, str]): + """ + See Spb2UNSPublisher#getSpbContext + """ + received_ctx = Spb2UNSPublisher.getSpbContext(group_id=group_id, + message_type=message_type, + edge_node_id=edge_node_id, + device_id=device_id) + assert received_ctx == expected_ctx, f"The context received: {received_ctx} was not as expected: {expected_ctx}" + + +@pytest.mark.parametrize( + "metrics_list", + [ + ([ + ( + "Temperature", # name + 1, # alias + sparkplug_b_pb2.Int32, # datatype + 32 # value + ), + ( + "Scale", # name + 2, # alias + sparkplug_b_pb2.String, # datatype + "Celsius" # value + ) + ]), + ([ + ( + "Pressure", # name + 3, # alias + sparkplug_b_pb2.Float, # datatype + 23.20 # value + ), + ( + "Scale", # name + 4, # alias + sparkplug_b_pb2.String, # datatype + "Bar" # value + ) + ]), + ]) +def testGetPayloadAndMetrics_Ddata(metrics_list: list[dict]): + """ + See Spb2UNSPublisher#getPayload + Spb2UNSPublisher#getMetricsListFromPayload + """ + sparkplug_message = uns_sparkplug_b_gen.Spb_Message_Generator() + sPBpayload = sparkplug_message.getDeviceDataPayload() + + for metric_data in metrics_list: + sparkplug_message.addMetric(payload=sPBpayload, + name=metric_data[0], + alias=metric_data[1], + type=metric_data[2], + value=metric_data[3]) + parsedPayload = Spb2UNSPublisher.getPayload(sPBpayload.SerializeToString()) + assert parsedPayload is not None, "parsed payload should not be none" + + assert parsedPayload == sPBpayload, f"parsed payload: {parsedPayload} is not matching original payload: {sPBpayload}" + + parsedMetrics_list: list = Spb2UNSPublisher.getMetricsListFromPayload( + sPBpayload.SerializeToString()) + assert len(parsedMetrics_list) == len(metrics_list) + for (parsed_metric, org_metric) in zip(parsedMetrics_list, metrics_list): + name = parsed_metric.name + alias = parsed_metric.alias + datatype = parsed_metric.datatype + value = getattr(parsed_metric, + Spb2UNSPublisher.SPB_DATATYPE_KEYS.get(datatype)) + + assert name == org_metric[0] and alias == org_metric[ + 1] and datatype == org_metric[2] + if (datatype == sparkplug_b_pb2.Float + or datatype == sparkplug_b_pb2.Double): + # Need to handle floating point issue in Python + assert round(value, 5) == round(org_metric[3], 5) + else: + assert value == org_metric[3] + + assert parsed_metric.timestamp is not None and parsed_metric.timestamp > 0 + + +@pytest.mark.parametrize("spbContext", [({ + "spBv1.0_group_id": "grp1", + "spBv1.0_message_type": "DDATA", + "spBv1.0_edge_node_id": "node1" +})]) +@pytest.mark.parametrize( + "parsed_message, tag_name, metric_value, metric_timestamp, isHistorical, expected_uns_message ", + [ + ( + None, + "Temp", + 23, + 1671028163, + False, + { #Test Set 1 -int, no historical data + "Temp": (23, 1671028163, False), + "timestamp": 1671028163, + "spBv1.0_group_id": "grp1", + "spBv1.0_message_type": "DDATA", + "spBv1.0_edge_node_id": "node1" + }), + ( + None, + "Grade", + "A", + 1671008100, + False, + { #Test Set 2 -String, no historical data + "Grade": ("A", 1671008100, False), + "timestamp": 1671008100, + "spBv1.0_group_id": "grp1", + "spBv1.0_message_type": "DDATA", + "spBv1.0_edge_node_id": "node1" + }), + ( + { + "Grade": ("A", 1671008100, + True), # Test Set 3 -String, with historical data + "timestamp": 1671008100, + "spBv1.0_group_id": "grp1", + "spBv1.0_message_type": "DDATA", + "spBv1.0_edge_node_id": "node1" + }, + "Grade", + "B", + 1671008200, + False, + { + "Grade": [("B", 1671008200, False), ("A", 1671008100, True)], + "timestamp": 1671008200, + "spBv1.0_group_id": "grp1", + "spBv1.0_message_type": "DDATA", + "spBv1.0_edge_node_id": "node1" + }), + ]) +def testExtractUNSMessageForTopic(parsed_message, tag_name, metric_value, + metric_timestamp, isHistorical, spbContext, + expected_uns_message): + uns_msg = Spb2UNSPublisher.extractUNSMessageForTopic( + parsed_message=parsed_message, + tag_name=tag_name, + metric_value=metric_value, + metric_timestamp=metric_timestamp, + isHistorical=isHistorical, + spbContext=spbContext) + assert uns_msg == expected_uns_message + + +@pytest.mark.parametrize("clean_session", [(True), (False)]) +@pytest.mark.parametrize("protocol", [(Uns_MQTT_ClientWrapper.MQTTv5), + (Uns_MQTT_ClientWrapper.MQTTv311)]) +@pytest.mark.parametrize("transport,host, port, tls", + [("tcp", "broker.emqx.io", 1883, None)]) +@pytest.mark.parametrize("qos", [(1), (2)]) +@pytest.mark.parametrize("reconnect_on_failure", [(True)]) +def test_publishToUNS_not_connected(clean_session, protocol, transport, host, + port, tls, qos, reconnect_on_failure): + """ + See Spb2UNSPublisher#init() + """ + uns_client: Uns_MQTT_ClientWrapper = Uns_MQTT_ClientWrapper( + client_id=f"spBv1.0_mapper_tester_{time.time()}", + clean_session=clean_session, + userdata=None, + protocol=protocol, + transport=transport, + reconnect_on_failure=reconnect_on_failure) + # Connection not made to broker + spg2unPub = Spb2UNSPublisher(uns_client) + all_uns_messages: dict = { + "a/b/c": { + "temperature": 28.0, + 123: "testing value", + "pressure": 10.12 + }, + "x/y/z": { + "temperature": 28.0, + 123: "testing value", + "pressure": 10.12 + } + } + with pytest.raises(ConnectionError): + spg2unPub.publishToUNS(all_uns_messages) + + +@pytest.mark.parametrize("clean_session", [(True), (False)]) +@pytest.mark.parametrize("protocol", [(Uns_MQTT_ClientWrapper.MQTTv5), + (Uns_MQTT_ClientWrapper.MQTTv311)]) +@pytest.mark.parametrize("transport,host, port, tls", + [("tcp", "broker.emqx.io", 1883, None)]) +@pytest.mark.parametrize("qos", [(1), (2)]) +@pytest.mark.parametrize("reconnect_on_failure", [(True)]) +def test_publishToUNS_connected(clean_session, protocol, transport, host, port, + tls, qos, reconnect_on_failure): + """ + See Spb2UNSPublisher#publishToUNS() + """ + msg_published = [] + uns_client: Uns_MQTT_ClientWrapper = Uns_MQTT_ClientWrapper( + client_id=f"spBv1.0_mapper_tester_{time.time()}", + clean_session=clean_session, + userdata=None, + protocol=protocol, + transport=transport, + reconnect_on_failure=reconnect_on_failure) + + all_uns_messages: dict = { + "a/b/c": { + "temperature": 28.0, + 123: "testing value", + "pressure": 10.12 + }, + "x/y/z": { + "temperature": 28.0, + 78945.12: "Can I have different keys?", + "my metric": 200 + } + } + + spg2unPub = Spb2UNSPublisher(uns_client) + + def on_publish(client, userdata, result): + msg_published.append(True) + print(f"{client} successfully published to {host} ") + if (len(msg_published) == len(all_uns_messages)): + client.disconnect() + + def on_connect(client, userdata, flags, rc, properties=None): + print(f"{client} successfully connected to {host} ") + spg2unPub.publishToUNS(all_uns_messages) + + uns_client.on_connect = on_connect + uns_client.on_publish = on_publish + try: + uns_client.run(host=host, port=port, tls=tls, topic="#", qos=qos) + uns_client.loop_forever() + except Exception as ex: + print(ex) + finally: + uns_client.loop_stop() + uns_client.disconnect() + + assert len(msg_published) == len( + all_uns_messages), "Not all messages were published" diff --git a/05_sparkplugb/test/test_uns_sparkplug_b_gen.py b/05_sparkplugb/test/test_uns_sparkplug_b_gen.py new file mode 100644 index 00000000..6a6ad363 --- /dev/null +++ b/05_sparkplugb/test/test_uns_sparkplug_b_gen.py @@ -0,0 +1,206 @@ +import inspect +import os +import sys +from types import SimpleNamespace +import pytest + +from google.protobuf.json_format import MessageToDict + +# From http://stackoverflow.com/questions/279237/python-import-a-module-from-a-folder +cmd_subfolder = os.path.realpath( + os.path.abspath( + os.path.join( + os.path.split(inspect.getfile(inspect.currentframe()))[0], '..', + 'src'))) +if cmd_subfolder not in sys.path: + sys.path.insert(0, cmd_subfolder) + sys.path.insert(1, os.path.join(cmd_subfolder, "uns_sparkplugb")) + sys.path.insert(2, + os.path.join(cmd_subfolder, "uns_sparkplugb", "generated")) + +import uns_sparkplug_b_gen +from uns_sparkplugb.generated import sparkplug_b_pb2 +# Dict containing value types as key value pair. +# Convert this to a SimpleNamespace to be able to access the attributes via dot notation +metric_dict = { + "int_value": None, + "long_value": None, + "float_value": None, + "double_value": None, + "string_value": None, + "bytes_value": None, + "boolean_value": None, + "template_value": None, + "datatype": None +} + + +@pytest.mark.parametrize( + "value,metric,factor, metric_value", + [(10, SimpleNamespace(**metric_dict), 0, 10), + (10, SimpleNamespace(**metric_dict), 8, 10), + (-10, SimpleNamespace(**metric_dict), 8, -10 + 2**8), + (10, SimpleNamespace(**metric_dict), 16, 10), + (-10, SimpleNamespace(**metric_dict), 16, -10 + 2**16), + (10, SimpleNamespace(**metric_dict), 32, 10), + (-10, SimpleNamespace(**metric_dict), 32, -10 + 2**32)]) +def test_setIntValueInMetric(value: int, metric: dict, factor: int, + metric_value: int): + uns_sparkplug_b_gen.setIntValueInMetric(value, metric, factor) + assert metric.int_value == metric_value, f"Expecting metric value to be: {metric_value}, but got {metric.int_value} " + for datatype in metric_dict: + if (datatype != "int_value"): + assert ( + metric.__dict__[datatype] is None + ), f"Datatype: {datatype} should be null in {metric} for all types except int_value" + + +@pytest.mark.parametrize( + "value,metric,factor, metric_value", + [(10, SimpleNamespace(**metric_dict), 0, 10), + (-10, SimpleNamespace(**metric_dict), 64, -10 + 2**64), + (10, SimpleNamespace(**metric_dict), 64, 10)]) +def test_setLongValueInMetric(value: int, metric: dict, factor: int, + metric_value: int): + # In python all long values are int + uns_sparkplug_b_gen.setLongValueInMetric(value, metric, factor) + assert metric.long_value == metric_value, f"Expecting metric value to be: {metric_value}, but got {metric.long_value} " + for datatype in metric_dict: + if (datatype != "long_value"): + assert ( + metric.__dict__[datatype] is None + ), f"Datatype: {datatype} should be null in {metric} for all types except long_value" + + +@pytest.mark.parametrize("value,metric, metric_value", [ + (10.0, SimpleNamespace(**metric_dict), 10.0), + (-10.0, SimpleNamespace(**metric_dict), -10.0), +]) +def test_setFloatValueInMetric(value: float, metric: dict, + metric_value: float): + uns_sparkplug_b_gen.setFloatValueInMetric(value, metric) + assert metric.float_value == metric_value, f"Expecting metric value to be: {metric_value}, but got {metric.float_value} " + for datatype in metric_dict: + if (datatype != "float_value"): + assert ( + metric.__dict__[datatype] is None + ), f"Datatype: {datatype} should be null in {metric} for all types except float_value" + + +@pytest.mark.parametrize("value,metric, metric_value", [ + (10.0, SimpleNamespace(**metric_dict), 10.0), + (-10.0, SimpleNamespace(**metric_dict), -10.0), +]) +def test_setDoubleValueInMetric(value: float, metric: dict, + metric_value: float): + uns_sparkplug_b_gen.setDoubleValueInMetric(value, metric) + assert metric.double_value == metric_value, f"Expecting metric value to be: {metric_value}, but got {metric.double_value} " + for datatype in metric_dict: + if (datatype != "double_value"): + assert ( + metric.__dict__[datatype] is None + ), f"Datatype: {datatype} should be null in {metric} for all types except double_value" + + +@pytest.mark.parametrize("value,metric, metric_value", [ + (True, SimpleNamespace(**metric_dict), True), + (False, SimpleNamespace(**metric_dict), False), +]) +def test_setBooleanValueInMetric(value: bool, metric: dict, + metric_value: bool): + uns_sparkplug_b_gen.setBooleanValueInMetric(value, metric) + assert metric.boolean_value == metric_value, f"Expecting metric value to be:{metric_value}, got:{metric.boolean_value}" + for datatype in metric_dict: + if (datatype != "boolean_value"): + assert ( + metric.__dict__[datatype] is None + ), f"Datatype: {datatype} should be null in {metric} for all types except boolean_value" + + +@pytest.mark.parametrize("value,metric, metric_value", [ + ("Test String1", SimpleNamespace(**metric_dict), "Test String1"), + ("""Test String2\nLine2""", SimpleNamespace(**metric_dict), + """Test String2\nLine2"""), +]) +def test_setStringValueInMetric(value: str, metric: dict, metric_value: str): + uns_sparkplug_b_gen.setStringValueInMetric(value, metric) + assert metric.string_value == metric_value, f"Expecting metric value to be: {metric_value}, but got {metric.string_value}" + for datatype in metric_dict: + if (datatype != "string_value"): + assert ( + metric.__dict__[datatype] is None + ), f"Datatype: {datatype} should be null in {metric} for all types except string_value" + + +@pytest.mark.parametrize("value,metric, metric_value", [ + (bytes("Test String1", "utf-8"), SimpleNamespace(**metric_dict), + bytes("Test String1", "utf-8")), + (bytes("""Test String2\nLine2""", "utf-8"), SimpleNamespace(**metric_dict), + bytes("""Test String2\nLine2""", "utf-8")), +]) +def test_setBytesValueInMetric(value: bytes, metric: dict, + metric_value: bytes): + uns_sparkplug_b_gen.setBytesValueInMetric(value, metric) + assert metric.bytes_value == metric_value, f"Expecting metric value to be: {metric_value}, but got {metric.string_value}" + for datatype in metric_dict: + if (datatype != "bytes_value"): + assert ( + metric.__dict__[datatype] is None + ), f"Datatype: {datatype} should be null in {metric} for all types except bytes_value" + + +def testSbp_Messages_SeqNum(): + sparkplug_message = uns_sparkplug_b_gen.Spb_Message_Generator() + # Test sequence + old_sequence = sparkplug_message.getSeqNum() + assert old_sequence == 0, f"Sequence number should start with 0 but stated with {old_sequence}" + + for count in range( + 270): # choose a number greater than 256 to test the counter reset + new_sequence = sparkplug_message.getSeqNum() + if old_sequence < 255: + assert old_sequence == new_sequence - 1, f"Loop {count}: Sequence not incrementing-> {old_sequence}:{new_sequence}" + else: + assert new_sequence == 0, f"Loop {count}: Sequence number should reset to 0 after 256 but got {new_sequence}" + old_sequence = new_sequence + + +def testSbp_Messages_BdSeqNum(): + # Test sequence + sparkplug_message = uns_sparkplug_b_gen.Spb_Message_Generator() + old_sequence = sparkplug_message.getBdSeqNum() + assert old_sequence == 0, f"Sequence number should start with 0 but stated with {old_sequence}" + + for count in range( + 260): # choose a number greater than 256 to test the counter reset + new_sequence = sparkplug_message.getBdSeqNum() + if old_sequence < 255: + assert old_sequence == new_sequence - 1, f"Loop {count}: Sequence not incrementing-> {old_sequence}:{new_sequence}" + else: + assert new_sequence == 0, f"Loop {count}: Sequence number should reset to 0 after 256 but got {new_sequence}" + old_sequence = new_sequence + + +def testSbp_Messages_getNodeDeathPayload(): + sparkplug_message = uns_sparkplug_b_gen.Spb_Message_Generator() + payload = sparkplug_message.getNodeDeathPayload() + payload_dict = MessageToDict(payload) + metric = payload_dict.get("metrics")[0] + assert metric.get("name") == "bdSeq" + assert metric.get("timestamp") is not None + assert metric.get("datatype") == sparkplug_b_pb2.Int64 + assert int(metric.get("longValue")) == 0 + + +def testSbp_Messages_getNodeBirthPayload(): + sparkplug_message = uns_sparkplug_b_gen.Spb_Message_Generator() + sparkplug_message.getNodeDeathPayload() + payload = sparkplug_message.getNodeBirthPayload() + payload_dict: dict = MessageToDict(payload) + metric = payload_dict.get("metrics")[0] + + assert payload.seq == 0 + assert metric.get("name") == "bdSeq" + assert metric.get("timestamp") is not None + assert metric.get("datatype") == sparkplug_b_pb2.Int64 + assert int(metric.get("longValue")) == 1 diff --git a/README.md b/README.md index 8b855c2c..2cba6d4b 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,7 @@ The overall architecture and the deployment setup is as follows * MQTT edge installed on K8s * Bridge between Factory1 and the Enterprise MQTT clusters * Graph DB installed and running on docker + * SparkplugB client to translate message from SparkPlug to UNS 1. K8s Cluster on the edge - Factory2 * K8s Cluster on the edge @@ -81,7 +82,14 @@ Some key limitations to bear in mind * microk8s is not available for every linux distribution. --- ### **MQTT Broker** -The backbone of the ***Unified Name Space*** is the MQTT broker. +The backbone of the ***Unified Name Space*** is the MQTT broker. +#### **Why MQTT** +The overall structure of the UNS is based on the hierarchical structure as defined in ISA-95 part 2. +> \/\/\/\\ + +The level at which the message is published has a direct implication on it's time sensitivity as well as guidance on being processed at the edge or on the cloud.
+![ISA-95 Part 2](./images/ISA-95-part2.png) + I evaluated and read the user guides of the following brokers (open source versions only). All three also provide commercial / enterprise versions which is recommended for more robust setup and professional support 1. [EMQX](https://www.emqx.io/) 2. [VERNEMQ](https://vernemq.com/) @@ -136,3 +144,13 @@ I choose to wite the client in Python even thought Python is not as performant a * In the OT space most professionals ( in my experience) were more familiar coding with Python than Go, C or Rust. Hence I hope this increases the adoptions and contributions from the community in further developing this tool * Should a team want to further optimize the code, given the readability and the inline comments in the code, they are hopefully able to rewrite the application in their choice of language * I wanted to learn Python + +### **Plugin / MQTT Client to translate SparkplugB messages to UNS Namespace** +Sparkplug consist of three primary features in its definition. +1. The first is the MQTT topic namespace definition. +2. The second is the definition of the order and flow of MQTT messages to and from various MQTT clients in the system. +3. The final is the payload data format which is called Sparkplug B. +As the messages are published in the Sparkplug Namespace and are packaged in protocol buffers, they are not visible in the UNS hierarchy which is based on ISA-95 part 2. + +The detailed description of the plugin can be found at [05_sparkplugb](./05_sparkplugb/Readme.md) +