Merge branch '25-adapt-pollution-computation' into 'main'

Resolve: Adapt pollution computation

Closes noi-techpark#25 and noi-techpark#28

See merge request u-hopper/projects/industrial/open-data-hub-bz/bdp-elaborations!21

pollution_v2/.gitignore

@@ -10,4 +10,5 @@ venv*
 celerybeat-schedule.db
 /sample_data/input/odh.db
 docker-compose.yaml.custom
+docker-compose.yaml.uh
 /sample_data/odh.db

pollution_v2/src/common/connector/common.py

@@ -317,7 +317,7 @@ def get_latest_measures(self, station: Optional[Station or str] = None,
         if len(where_conds) > 0:
             query_params["where"] = f'and({",".join(where_conds)})'
 
-        logger.info(f"Retrieving latest measures on [{type(self).__name__}] with where [{query_params['where']}]")
+        logger.debug(f"Retrieving latest measures on [{type(self).__name__}] with where [{query_params['where']}]")
 
         raw_measures = self._get_result_list(
             path=f"/v2/flat,node/{self._station_type}/{','.join(self._measure_types)}/latest",
@@ -350,7 +350,7 @@ def get_measures(self, from_date: datetime, to_date: datetime, station: Optional
         if len(where_conds) > 0:
             query_params["where"] = f'and({",".join(where_conds)})'
 
-        logger.info(f"Retrieving measures on [{type(self).__name__}] from date [{iso_from_date}] "
+        logger.debug(f"Retrieving measures on [{type(self).__name__}] from date [{iso_from_date}] "
                     f"to date [{iso_to_date}] with where [{query_params['where']}]")
 
         raw_measures = self._get_result_list(
@@ -399,6 +399,11 @@ def _post_request(self, path: str, raw_data: dict or list,
         while call_counter <= self._requests_max_retries:
             call_counter += 1
             try:
+                logger.debug("posting...")
+                logger.debug(f"where: {self._base_writer_url}{path}")
+                logger.debug(f"what:  {raw_data}")
+                logger.debug(f"with:  {self._get_authentication_header(token)}")
+                logger.debug(f"param: {query_params}")
                 response = requests.post(
                     f"{self._base_writer_url}{path}",
                     json=raw_data,
@@ -408,12 +413,16 @@ def _post_request(self, path: str, raw_data: dict or list,
                 )
                 if response.status_code in [200, 201]:
                     try:  # ODH seems that id not returning a json for post requests but a string
+                        logger.debug(f"Response code is {response.status_code}, response is {response.json()}")
                         return response.json()
                     except requests.exceptions.JSONDecodeError:
+                        logger.debug(f"Response code is {response.status_code}, error decoding json: {response.text}")
                         return response.text
                 else:
+                    logger.debug(f"Response code is {response.status_code}, considered as error!")
                     raise ApiException(message=response.content.decode("utf-8"), code=response.status_code)
             except ApiException as e:
+                logger.exception("API Exception in getting data from ODH", exc_info=e)
                 raise e
             except Exception as e:
                 logger.exception("Exception in getting data from ODH", exc_info=e)

pollution_v2/src/common/data_model/pollution.py

@@ -12,6 +12,7 @@
 from common.data_model.common import VehicleClass, MeasureCollection, Measure, Provenance, DataType
 from common.data_model.entry import GenericEntry
 from common.data_model.station import TrafficSensorStation
+from common.settings import DATATYPE_PREFIX
 
 
 class PollutantClass(Enum):
@@ -49,7 +50,7 @@ def get_data_types() -> List[DataType]:
         data_types = []
         for vehicle in VehicleClass:
             for pollutant in PollutantClass:
-                data_types.append(DataType(f"{vehicle.name}-{pollutant.name}-emissions",
+                data_types.append(DataType(f"{DATATYPE_PREFIX}{vehicle.name}-{pollutant.name}-emissions",
                                            f"{vehicle.value} emissions of {pollutant.name}", "total", "g/km", {}))
         return data_types
 
@@ -72,7 +73,7 @@ def build_from_entries(pollution_entries: List[PollutionEntry],
         for pollution_entry in pollution_entries:
             pollution_measures.append(PollutionMeasure(
                 station=pollution_entry.station,
-                data_type=data_types_dict[f"{pollution_entry.vehicle_class.name}-{pollution_entry.entry_class.name}-emissions"],
+                data_type=data_types_dict[f"{DATATYPE_PREFIX}{pollution_entry.vehicle_class.name}-{pollution_entry.entry_class.name}-emissions"],
                 provenance=provenance,
                 period=pollution_entry.period,
                 transaction_time=None,

pollution_v2/src/common/data_model/validation.py

@@ -12,6 +12,7 @@
 from common.data_model.common import VehicleClass, MeasureCollection, Measure, Provenance, DataType
 from common.data_model.entry import GenericEntry
 from common.data_model.traffic import TrafficSensorStation, TrafficEntry
+from common.settings import DATATYPE_PREFIX
 
 
 class ValidationTypeClass(Enum):
@@ -46,7 +47,7 @@ def get_data_types() -> List[DataType]:
         for vehicle in VehicleClass:
             for validation_type in ValidationTypeClass:
                 data_types.append(
-                    DataType(f"{vehicle.name}-{validation_type.name}", f"{vehicle.value} is {validation_type.name}",
+                    DataType(f"{DATATYPE_PREFIX}{vehicle.name}-{validation_type.name}", f"{vehicle.value} is {validation_type.name}",
                              "Validation", "-", {}))
         return data_types
 
@@ -73,7 +74,7 @@ def build_from_entries(validation_entries: List[ValidationEntry],
                 validation_measures.append(ValidationMeasure(
                     station=validation_entry.station,
                     data_type=data_types_dict[
-                        f"{validation_entry.vehicle_class.name}-{validation_entry.entry_class.name}"],
+                        f"{DATATYPE_PREFIX}{validation_entry.vehicle_class.name}-{validation_entry.entry_class.name}"],
                     provenance=provenance,
                     period=validation_entry.period,
                     transaction_time=None,