From 8c3e14ead9a25d402c64f96bebf9f07f94a47a93 Mon Sep 17 00:00:00 2001 From: Abdo-Gaber Date: Mon, 10 Mar 2025 13:03:52 +0100 Subject: [PATCH] Add Data Recording Application v1.0 (main app, SigMF Interface, Config files, Sync validation) The data recording App includes the following files: - The main app: data_recording_app_v1.0.py that has the data control service, interact with tracers, and data sync and conversion service - The configuration file: common/utils/data_recording/config/config_data_recording.json - The wireless dictionary file: common/utils/data_recording/config/wireless_link_parameter_map.yaml - The SigMF Interface (common/utils/data_recording/lib/sigmf_interface.py) to store recorded data and meta-data in SigMF format - A simple script (common/utils/data_recording/sync_validation_demo.py) to validate that the recorded bits from gNB and UE are in Sync - For more info look to doc/data_recording.md A squashed commit further updates: - Enhance Data Recording App structure - Support the new state machine of T-Tracers - Add new files for Config Interface, sync Service, def of data recording messages, and common utilties - Make part of logging on Terminal configurable --- .../config/config_data_recording.json | 84 ++ .../config/wireless_link_parameter_map.yaml | 375 ++++++++ .../data_recording/data_recording_app_v1.0.py | 829 ++++++++++++++++++ common/utils/data_recording/lib/__init__.py | 32 + .../utils/data_recording/lib/common_utils.py | 74 ++ .../data_recording/lib/config_interface.py | 150 ++++ .../lib/data_recording_messages_def.py | 107 +++ .../data_recording/lib/sigmf_interface.py | 580 ++++++++++++ .../utils/data_recording/lib/sync_service.py | 150 ++++ common/utils/data_recording/requirements.txt | 5 + .../data_recording/sync_validation_demo.py | 150 ++++ 11 files changed, 2536 insertions(+) create mode 100644 common/utils/data_recording/config/config_data_recording.json create mode 100644 common/utils/data_recording/config/wireless_link_parameter_map.yaml create mode 100644 common/utils/data_recording/data_recording_app_v1.0.py create mode 100644 common/utils/data_recording/lib/__init__.py create mode 100644 common/utils/data_recording/lib/common_utils.py create mode 100644 common/utils/data_recording/lib/config_interface.py create mode 100644 common/utils/data_recording/lib/data_recording_messages_def.py create mode 100644 common/utils/data_recording/lib/sigmf_interface.py create mode 100644 common/utils/data_recording/lib/sync_service.py create mode 100644 common/utils/data_recording/requirements.txt create mode 100644 common/utils/data_recording/sync_validation_demo.py diff --git a/common/utils/data_recording/config/config_data_recording.json b/common/utils/data_recording/config/config_data_recording.json new file mode 100644 index 0000000000..2d4387d664 --- /dev/null +++ b/common/utils/data_recording/config/config_data_recording.json @@ -0,0 +1,84 @@ +{ + "id": 0, + "test_config": { + "test_scenario_id": "", + "test_name": "phy_ul_bler_test", + "test_mode": "rf_real_time", + "dut_type": "trad_rx" + }, + "environment_emulation": { + "target_link_config": { + "ran_config": { + "uplink": { + "ul_used_signal_BW_MHz": 2.16 + } + }, + "wireless_channel": { + "downlink": { + "type": "rf_cable" + }, + "uplink": { + "type": "statistical_var", + "statistical": { + "var_power_delay_profile": "tdl_d", + "speed_mps": 5.0, + "delay_spread_ns": 50.0, + "snr_db": 20.0, + "predef_channel_profile": null + } + }, + "carrierFreqValueList_hz": [ + 3319680000.0 + ], + "ul_used_signal_BW_MHz": 2.16 + } + } + }, + "data_recording_config": { + "data_storage_path": "/home/user/workarea/oai_recorded_data/", + "data_file_format": "SigMF", + "enable_saving_tracer_messages_sigmf": true, + "num_records": 5, + "t_tracer_message_definition_file": "../T/T_messages.txt", + "parameter_map_file": "config/wireless_link_parameter_map.yaml", + "start_frame_number": 5, + "base_station": { + "requested_tracer_messages": [ + "GNB_PHY_UL_FD_PUSCH_IQ", + "GNB_PHY_UL_FD_DMRS", + "GNB_PHY_UL_FD_CHAN_EST_DMRS_POS", + "GNB_PHY_UL_FD_CHAN_EST_DMRS_INTERPL", + "GNB_PHY_UL_PAYLOAD_RX_BITS" + ], + "meta_data":{ + "num_rx_antennas": 1, + "tx_gain": 48.0, + "rx_gain": 30.0, + "hw_type": "USRP X410", + "hw_subtype": "ZBX", + "seid": "328AB35" + } + }, + "user_equipment": { + "requested_tracer_messages": [ + "UE_PHY_UL_SCRAMBLED_TX_BITS", + "UE_PHY_UL_PAYLOAD_TX_BITS" + ], + "meta_data":{ + "num_tx_antennas": 1, + "tx_gain": 48.0, + "rx_gain": 40.0, + "hw_type": "USRP X410", + "hw_subtype": "ZBX", + "seid": "323F75F" + } + }, + "common_meta_data": { + "sample_rate": 61440000.0, + "bandwidth": 40000000.0, + "clock_reference": "external" + }, + "tracer_service_baseStation_address": "127.0.0.1:2021", + "tracer_service_userEquipment_address": "127.0.0.1:2023" + } +} diff --git a/common/utils/data_recording/config/wireless_link_parameter_map.yaml b/common/utils/data_recording/config/wireless_link_parameter_map.yaml new file mode 100644 index 0000000000..deab34434c --- /dev/null +++ b/common/utils/data_recording/config/wireless_link_parameter_map.yaml @@ -0,0 +1,375 @@ +#/* +# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more +# * contributor license agreements. See the NOTICE file distributed with +# * this work for additional information regarding copyright ownership. +# * The OpenAirInterface Software Alliance licenses this file to You under +# * the OAI Public License, Version 1.1 (the "License"); you may not use this file +# * except in compliance with the License. +# * You may obtain a copy of the License at +# * +# * http://www.openairinterface.org/?page_id=698 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# *------------------------------------------------------------------------------- +# * For more information about the OpenAirInterface (OAI) Software Alliance: +# * contact@openairinterface.org +# */ +#--------------------------------------------------------------------- +# file common/utils/data_recording/config/wireless_link_parameter_map.yaml +# brief Wireless Link Parameter Map Dictionary +# author Abdo Gaber +# date 2024 +# version 1.0 +# company Emerson, NI Test and Measurement +# email: +# note +# warning + + +# Wireless Link Parameter Map +# Components: +# Transmitter: list of waveform configuration based on standard +# Parameters pair is given via this parameter map dictionary (Parameter name in Waveform creator vs that in the API ~ SigMF meta-data) +# Having a separate dictionary ease of adoption in case of: +# - Adding new parameters +# - Use different waveform creator that has Different naming scheme +# - Define your own customized standard or new wireless system +# - All changes are limited to parameter map dictionary +# the parameter name of waveform generator has the following style: +# StandardName_WaveformGenerator_parameter, i.e. +# if standard = 5gnr, WaveformGenerator: ni_rfmx_rfws, then the parameter name: 5gnr_ni_rfmx_rfws_parameter +# ===================================================================================== +# ======== list of parameter pairs between waveform config and API ~ SigMF Meta-Data === +# ===================================================================================== + +# List of waveform generators that generate waveforms +# key: +# standard: "name" +# generator: "link or text ... etc" +# key: Used to create parameter names in wireless_link_parameter_map +# Recommended name for each waveform generator to be given as: standard_WaveformGenerator +# Example: 5g NR waveform created using RFmx with RFWS config file: "5gnr_ni_rfmx_rfws" +# Note: The suffix "_parameter" will be added automatically +# standard name: occur in meta-data +# generator: generator description to occur in meta-data + +waveform_generator: + # waveform_source: standard + 5gnr_oai: # Generated using OAI + standard: "5gnr" + generator: "OpenAirInterface: https://www.openairinterface.org/" + 5gnr_ni_rfmx_rfws: # Generated using NI RFmx Tool + standard: "5gnr" + generator: "NI RFmx Waveform Creator: https://www.ni.com/en-ca/shop/wireless-design-test/application-software-for-wireless-design-test-category/what-is-rfmx.html" +transmitter: + # 5G NR subfield + 5gnr: + # frame number + - sigmf_parameter_name: "frame" + description: "5G NR frame number" + 5gnr_oai_parameter: + name: "frame" + # slot number + - sigmf_parameter_name: "slot" + description: "5G NR slot number" + 5gnr_oai_parameter: + name: "slot" + # bandwidth + - sigmf_parameter_name: "bandwidth" + description: "5G NR bandwidth" + 5gnr_ni_rfmx_rfws_parameter: + name: "Bandwidth (Hz)" + 5gnr_oai_parameter: + name: "bandwidth" + # Frame structure + - sigmf_parameter_name: "frame_structure" + description: "5G NR duplexing scheme FDD, TDD, user defined" + 5gnr_ni_rfmx_rfws_parameter: + name: "Duplex Scheme" # not exact, since it depends if the waveform is for UL or DL + 5gnr_oai_parameter: + name: "frame_type" + value_map: # simu value : SigMF value + 0: "fdd" + 1: "tdd" + # Link direction + - sigmf_parameter_name: "link_direction" + description: "Link direction" + 5gnr_ni_rfmx_rfws_parameter: + name: "Link Direction" + # Frequency Range + - sigmf_parameter_name: "frequency_range" + description: "Frequency Range" + 5gnr_ni_rfmx_rfws_parameter: + name: "Frequency Range" + value_map: # simu value : SigMF value + "Range 1": "fr1" + "Range 2": "fr2" + 5gnr_oai_parameter: + name: "freq_range" + value_map: # simu value : SigMF value + 0: "fr1" + 1: "fr2" + # Subcarrier spacing in Hz + - sigmf_parameter_name: "subcarrier_spacing" + description: "Subcarrier spacing in Hz" + 5gnr_ni_rfmx_rfws_parameter: + name: Subcarrier Spacing (Hz) + 5gnr_oai_parameter: + name: subcarrier_spacing + value_map: # simu value : SigMF value + 0: 15000 + 1: 30000 + 2: 60000 + 3: 120000 + # Sampling rate in Hz + - sigmf_parameter_name: "sample_rate" + description: "Sampling rate in Hz" + 5gnr_ni_rfmx_rfws_parameter: + name: "sample_rate_hz" + 5gnr_oai_parameter: + name: "sample_rate" + # Cyclic prefix mode + - sigmf_parameter_name: "cp_mode" + description: "Cyclic prefix mode" + 5gnr_ni_rfmx_rfws_parameter: + name: "Cyclic Prefix Mode" + 5gnr_oai_parameter: + name: "cyclic_prefix" + value_map: # simu value : SigMF value + 0: "normal" + 1: "extended" + # cell ID + - sigmf_parameter_name: "cell_id" + description: "cell ID" + 5gnr_ni_rfmx_rfws_parameter: + name: "Cell ID" + 5gnr_oai_parameter: + name: "Nid_cell" + # rnti + - sigmf_parameter_name: "rnti" + description: "RNTI" + 5gnr_ni_rfmx_rfws_parameter: + name: "rnti" + 5gnr_oai_parameter: + name: "rnti" + # Symbols per slot + - sigmf_parameter_name: "symbols_per_slot" + description: "Symbols per slot" + 5gnr_oai_parameter: + name: "symbols_per_slot" + # PUSCH: content + - sigmf_parameter_name: "pusch:content" + description: "PUSCH: content" + # PUSCH: channel code + - sigmf_parameter_name: "pusch:channel_code" + description: "PUSCH: channel code" + # PUSCH: number of PRBs + - sigmf_parameter_name: "pusch:num_prb" + description: "PUSCH Resource Block Size" + 5gnr_oai_parameter: + name: "rb_size" + # PUSCH RB Start index + - sigmf_parameter_name: "pusch:start_prb" + description: "PUSCH Resource Block Start" + 5gnr_oai_parameter: + name: "rb_start" + # PUSCH MAPPING type + - sigmf_parameter_name: "pusch:mapping_type" + description: "PUSCH Mapping Type" + 5gnr_oai_parameter: + name: "mappingType" + value_map: # simu value : SigMF value + "NR_PUSCH_TimeDomainResourceAllocation__mappingType_typeA": "A" + "NR_PUSCH_TimeDomainResourceAllocation__mappingType_typeB": "B" + # PUSCH: OFDM symbol start index within slot + - sigmf_parameter_name: "pusch:start_ofdm_symbol" + description: "PUSCH: OFDM symbol start index within slot" + 5gnr_oai_parameter: + name: "start_symbol_index" + # PUSCH: number of OFDM symbols in one slot + - sigmf_parameter_name: "pusch:num_ofdm_symbols" + description: "PUSCH: number of OFDM symbols in one slot" + 5gnr_oai_parameter: + name: "nr_of_symbols" + # PUSCH: modulation + - sigmf_parameter_name: "pusch:modulation" + description: "PUSCH: modulation" + 5gnr_ni_rfmx_rfws_parameter: + name: "UL Modulation Type" + value_map: # simu value : SigMF value + "8PSK": "8_psk" + "QPSK": "qpsk" + "QAM16": "16_qam" + "QAM64": "64_qam" + "QAM256": "256_qam" + "QAM1024": "1024_qam" + "Standard": "" + 5gnr_oai_parameter: + name: "qam_mod_order" + value_map: # simu value : SigMF value + 2 : "qpsk" + 4 : "16_qam" + 6 : "64_qam" + 8 : "256_qam" + "Standard": "" + # PUSCH MCS Index + - sigmf_parameter_name: "pusch:mcs" + description: "PUSCH MCS Index" + 5gnr_oai_parameter: + name: "mcs_index" + # PUSCH MCS Index Table + - sigmf_parameter_name: "pusch:mcs_table_index" + description: "PUSCH MCS Table Index" + 5gnr_oai_parameter: + name: "mcs_table" + # PUSCH: number of transmission layers + - sigmf_parameter_name: "pusch:num_layer" + description: "PUSCH: number of transmission layers" + 5gnr_oai_parameter: + name: "nrOfLayers" + # Transform Precoding + - sigmf_parameter_name: "pusch:transform_precoding" + description: "PUSCH Transform Precoding" + 5gnr_oai_parameter: + name: "transform_precoding" + value_map: # simu value : SigMF value + 0: "enabled" + 1: "disabled" + # PUSCH DMRS: content + - sigmf_parameter_name: "pusch_dmrs:content" + description: "PUSCH DMRS: content" + #"none": "none" + #"random_qpsk": "random_qpsk" + #"compliant": "compliant" + # PUSCH DMRS: resource map configuration + - sigmf_parameter_name: "pusch_dmrs:resource_map_config" + description: "PUSCH DMRS: resource map configuration" + 5gnr_oai_parameter: + name: "dmrs_config_type" + value_map: + 0: "5g nr type 1" + 1: "5g nr type 2" + # PUSCH DMRS: OFDM symbol start index within slot + - sigmf_parameter_name: "pusch_dmrs:start_ofdm_symbol" + description: "PUSCH DMRS: OFDM symbol start index within slot" + 5gnr_oai_parameter: + # to be derived from this OAI parameter + name: "ul_dmrs_symb_pos" + # PUSCH DMRS: DMRS duration in number of OFDM symbols + - sigmf_parameter_name: "pusch_dmrs:duration_num_ofdm_symbols" + description: "PUSCH DMRS: DMRS duration in number of OFDM symbols" + 5gnr_oai_parameter: + # to be derived from this OAI parameter + name: "ul_dmrs_symb_pos" + # PUSCH DMRS: OFDM symbol indices within resource grid + - sigmf_parameter_name: "pusch_dmrs:ofdm_symbol_idx" + description: "PUSCH DMRS: OFDM symbol indices within resource grid" + 5gnr_oai_parameter: + # to be derived from this OAI parameter + name: "ul_dmrs_symb_pos" + # PUSCH DMRS: number of additional PUSCH DMRS positions + - sigmf_parameter_name: "pusch_dmrs:num_add_positions" + description: "PUSCH DMRS: number of additional PUSCH DMRS positions" + 5gnr_oai_parameter: + # to be derived from this OAI parameter + name: "ul_dmrs_symb_pos" + # PUSCH DMRS: antenna port + - sigmf_parameter_name: "pusch_dmrs:antennna_port" + description: "PUSCH DMRS: antenna port" + 5gnr_oai_parameter: + name: "dmrs_port" + # PUSCH DMRS: scrambler initialization + - sigmf_parameter_name: "pusch_dmrs:nscid" + description: "PUSCH DMRS: scrambler initialization" + 5gnr_oai_parameter: + name: "dmrs_nscid" + # number of rx antennas + - sigmf_parameter_name: "num_rx_antennas" + description: "Number of Rx antennas" + 5gnr_oai_parameter: + name: "nb_antennas_rx" + # number of tx antennas + - sigmf_parameter_name: "num_tx_antennas" + description: "Number of Tx antennas" + 5gnr_oai_parameter: + name: "nb_antennas_tx" + +channel: + # Signal-to-noise ratio (Es/N0) in dB + - sigmf_parameter_name: "snr_esn0_db" + description: "Signal-to-noise ratio (Es/N0) in dB" + # Signal-to-noise ratio (Eb/N0) in dB + - sigmf_parameter_name: "snr_ebn0_db" + description: "Signal-to-noise ratio (Eb/N0) in dB" + # Channel model + - sigmf_parameter_name: "channel_model" + description: "Fading channel model" + # Number of channel taps + - sigmf_parameter_name: "num_channel_taps" + description: "Number of channel taps" + # Channel attenuation level in dB + - sigmf_parameter_name: "channel_attenuation_db" + description: "Channel attenuation level in dB" + # Nominal delay spread in s + - sigmf_parameter_name: "delay_spread" + description: "Nominal delay spread in s" + # UE speed in m/s + - sigmf_parameter_name: "speed" + description: "UE speed in m/s" + # Carrier frequency in Hz + - sigmf_parameter_name: "carrier_frequency" + description: "Carrier frequency in Hz" +receiver: + # Frequency domain interpolation method for channel estimation + - sigmf_parameter_name: "channel_estimation_freq_interpolation" + description: "Frequency domain interpolation method for channel estimation" + # Time domain interpolation method for channel estimation + - sigmf_parameter_name: "channel_estimation_time_interpolation" + description: "Time domain interpolation method for channel estimation" + # Channel equalization method + - sigmf_parameter_name: "channel_equalization_method" + description: "Channel equalization method" + # Symbol demapping method + - sigmf_parameter_name: "symbol_demapping_method" + description: "Symbol demapping method" + # Manufacturer of the hardware used to receive the signal + - sigmf_parameter_name: "manufacturer" + description: "Manufacturer" + # Unique ID of the emitter, USRP serial number --> usrp_mboard_serial + - sigmf_parameter_name: "seid" + description: "Emitter ID (USRP serial number)" + # For USRP, use “daughterboard ID, a string read out from the device shortened by the hex ID e.g. ‘(0x0085)’ + - sigmf_parameter_name: "hw_subtype" + description: "Hardware subtype (for USRP: daughterboard ID)" + # RX sync reference source ("internal", "external", "gpsdo") + - sigmf_parameter_name: "clock_reference" + description: "RX clock reference" + required_field: false + # Estimated SNR in dB + - sigmf_parameter_name: "snr_estimate_db" + description: "SNR estimate in dB" + required_field: false + # Result of coarse PSS-based timing estimation (used for synchronization) + - sigmf_parameter_name: "timing_estimate_coarse" + description: "Coarse timing estimate in samples" + required_field: false" + # Result of coarse PSS-based frequency offset estimation (used for synchronization) + - sigmf_parameter_name: "cfo_estimate_coarse" + description: "Coarse frequency estimate in Hz" + required_field: false + # Result of fine CRS-based frequency offset estimation (used for synchronization) + - sigmf_parameter_name: "cfo_estimate_fine" + description: "Fine frequency estimate in Hz" + required_field: false + # Number of transmitted payload bits + - sigmf_parameter_name: "num_payload_bits" + description: "Number of transmitted payload bits" + required_field: false + # Number of bit errors when applying traditional RX processing + - sigmf_parameter_name: "bit_errors_trad_rx" + description: "Bit errors" + required_field: false \ No newline at end of file diff --git a/common/utils/data_recording/data_recording_app_v1.0.py b/common/utils/data_recording/data_recording_app_v1.0.py new file mode 100644 index 0000000000..53b7555b4b --- /dev/null +++ b/common/utils/data_recording/data_recording_app_v1.0.py @@ -0,0 +1,829 @@ +# /* +# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more +# * contributor license agreements. See the NOTICE file distributed with +# * this work for additional information regarding copyright ownership. +# * The OpenAirInterface Software Alliance licenses this file to You under +# * the OAI Public License, Version 1.1 (the "License"); you may not use this file +# * except in compliance with the License. +# * You may obtain a copy of the License at +# * +# * http://www.openairinterface.org/?page_id=698 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# *------------------------------------------------------------------------------- +# * For more information about the OpenAirInterface (OAI) Software Alliance: +# * contact@openairinterface.org +# */ +# --------------------------------------------------------------------- +# file common/utils/data_recording/data_recording_app_v1.0.py +# brief main application of synchronized real-time data recording +# author Abdo Gaber +# date 2025 +# version 1.0 +# company Emerson, NI Test and Measurement +# email: +# note +# warning + +import sysv_ipc as ipc +import struct +import time +from datetime import datetime +from termcolor import colored +import numpy as np +import json +import concurrent.futures +# from concurrent import futures +import threading +# import related functions +from lib import sigmf_interface +# import library functions +from lib import sync_service +from lib import data_recording_messages_def +from lib import common_utils +from lib import config_interface + +DEBUG_WIRELESS_RECORDED_DATA = True +DEBUG_BUFFER_READING = False + +# globally applicable metadata +global_info = { + "author": "Abdo Gaber", + "description": "Synchronized Real-Time Data Recording", + "timestamp": 0, + "collection_file_prefix": "data-collection", # collection file name prefix "deap-rx + str(...)" + "collection_file": "", # Reserved to be created in the code: “data-collection_rec-0_TIME-STAMP” + "datetime_offset": "", # datetime offset between current location and UTC/Zulu timezone + # Example: "+01:00" for Berlin, Germany + "save_config_data_recording_app_json": True, + "waveform_generator": "5gnr_oai", + "extensions": {}, +} + +# Supported OAI Trace messages +# UL receiver messages +# gNB IQ Msgs: "GNB_PHY_UL_FD_PUSCH_IQ", "GNB_PHY_UL_FD_DMRS", "GNB_PHY_UL_FD_CHAN_EST_DMRS_POS", +# "GNB_PHY_UL_FD_CHAN_EST_DMRS_INTERPL" +# gNB BITS Msgs: "GNB_PHY_UL_PAYLOAD_RX_BITS" +# UE BITS Msgs: "UE_PHY_UL_SCRAMBLED_TX_BITS", "UE_PHY_UL_PAYLOAD_TX_BITS" + +supported_oai_tracer_messages = { + # gNB messages + "GNB_PHY_UL_FD_PUSCH_IQ": { + "file_name_prefix": "rx-fd-data", + "scope": "gNB", + "description": "Frequency-domain RX data", + "serialization_scheme": ["subcarriers", "ofdm_symbols"], + }, + "GNB_PHY_UL_FD_DMRS": { + "file_name_prefix": "tx-pilots-fd-data", + "scope": "gNB", + "description": "Frequency-domain TX PUSCH DMRS data", + "serialization_scheme": ["subcarriers", "ofdm_symbols"], + }, + "GNB_PHY_UL_FD_CHAN_EST_DMRS_POS": { + "file_name_prefix": "raw-ce-fd-data", + "scope": "gNB", + "description": "Frequency-domain raw channel estimates (at DMRS positions)", + "serialization_scheme": ["subcarriers", "ofdm_symbols"], + }, + "GNB_PHY_UL_FD_CHAN_EST_DMRS_INTERPL": { + "file_name_prefix": "raw-inter-ce-fd-data", + "scope": "gNB", + "description": "Interpolcated Frequency-domain raw channel estimates", + "serialization_scheme": ["subcarriers", "ofdm_symbols"], + }, + "GNB_PHY_UL_PAYLOAD_RX_BITS": { + "file_name_prefix": "rx-payload-bits", + "scope": "gNB", + "description": "Received PUSCH payload bits", + "serialization_scheme": ["bits", "subcarriers", "ofdm_symbols"], + }, + # UE messages + "UE_PHY_UL_SCRAMBLED_TX_BITS": { + "file_name_prefix": "tx-scrambled-bits", + "scope": "UE", + "description": "Transmitted scrambled PUSCH bits", + "serialization_scheme": ["bits", "subcarriers", "ofdm_symbols"], + }, + "UE_PHY_UL_PAYLOAD_TX_BITS": { + "file_name_prefix": "tx-payload-bits", + "scope": "UE", + "description": "Transmitted PUSCH payload bits", + "serialization_scheme": ["bits", "subcarriers", "ofdm_symbols"], + }, +} +# ------------------------------------------- +# System configuration: gNB +project_id_gnb = 2335 +read_shm_path_gnb = "/tmp/gnb_app1" +write_shm_path_gnb = "/tmp/gnb_app2" + +# System configuration: UE +project_id_ue = 2336 +read_shm_path_ue = "/tmp/ue_app1" +write_shm_path_ue = "/tmp/ue_app2" + + +# initialize shared memory +def attach_shm(shm_path, project_id): + key = ipc.ftok(shm_path, project_id) + shm = ipc.SharedMemory(key, 0, 0) + # I found if we do not attach ourselves + # it will attach as ReadOnly. + shm.attach(0, 0) + return shm + + +def detach_shm(shm): + try: + shm.detach() + print("Shared memory detached successfully.") + except ipc.ExistentialError: + print("Shared memory segment does not exist.") + + +def remove_shm(shm): + try: + shm.remove() + print("Shared memory removed successfully.") + except ipc.ExistentialError: + print("Shared memory segment does not exist.") + + +# check data if avalible in the shared memory +def is_data_available_in_memory(shm, bufIdx, general_message_header_length, timeout=20): + start_time = time.time() + while True: + buf = shm.read(bufIdx + general_message_header_length) + n_bytes = sum(buf) + print("Data Recording App: Waiting for Measurements!") + if n_bytes > 0: + print("There is data in memory, n_bytes: ", n_bytes) + return True + if (time.time() - start_time) > timeout: + break + time.sleep(1) + return False + + +# Read data from Shared memory based Data Conversion Service message structure +def read_data_from_shm(shm, bufIdx, tracer_msgs_identities): + # print buffer index + if DEBUG_BUFFER_READING: + print("Buffer Index: ", bufIdx) + # get general message header list + general_msg_header_list, general_message_header_length = data_recording_messages_def.get_general_msg_header_list() + buf = shm.read(bufIdx + general_message_header_length) + n_bytes = sum(buf) + if n_bytes == 0: + raise Exception('ERROR: No data available in memory') + msg_id = struct.unpack(' timeout_sync: + raise Exception( + "ERROR: Data Recording NO Sync Found, check Tracer Services if they are connected!") + + # Get gNB data based on the sync data + bufIdx = 0 + while True: + time.sleep(0.0035) + gnb_bufIdx = bufIdx + captured_data, bufIdx = read_data_from_shm( + shm_reading_gnb, bufIdx, tracer_msgs_identities) + if (captured_data["frame"] == sync_info["frame"] + and captured_data["slot"] == sync_info["slot"]): + break + + # Read Synchronized data between gNB and UE + while True: # read all records + print("\nRecord number: ", record_idx) + if DEBUG_BUFFER_READING: + print(f"Buffer Index gNB: {gnb_bufIdx}, Buffer Index UE: {ue_bufIdx}") + # wait for the next record + # To do: check if we need to add exta waiting times between different events in case of + # data streaming via network such as on UE side or gNB side + time.sleep(0.0035) # 2.3 ms = latency of T tracer to capture data from the RAN + + collected_metafiles = [] + # Read data from gNB T-tracer Application + for idx in range(gnb_args.num_requested_tracer_msgs): + time.sleep(0.0015) + if DEBUG_WIRELESS_RECORDED_DATA: + print(f"\nRecord number: {record_idx}, Reading MSG data ", idx) + captured_data, gnb_bufIdx = read_data_from_shm( + shm_reading_gnb, gnb_bufIdx, tracer_msgs_identities) + # drive the collection file time stamp from the first message per record + if idx == 0: + # Get time stamp + time_stamp_ms, time_stamp_ms_file_name = ( + sigmf_interface.time_stamp_formating( + captured_data["time_stamp"], global_info["datetime_offset"])) + global_info["collection_file"] = ( + global_info["collection_file_prefix"] + + "-rec-" + + str(record_idx) + + "-" + + str(time_stamp_ms_file_name) + ) + global_info["timestamp"] = time_stamp_ms + + # Write data into files with the given format + if config_meta_data["data_recording_config"]["enable_saving_tracer_messages_sigmf"]: + collected_metafiles.append( + sigmf_interface.write_recorded_data_to_sigmf( + captured_data, config_meta_data, global_info, record_idx)) + + # Read data from UE T-tracer Application + for idx in range(ue_args.num_requested_tracer_msgs): + time.sleep(0.0015) + if DEBUG_WIRELESS_RECORDED_DATA: + print(f"\nRecord number: {record_idx}, Reading MSG data ", idx) + captured_data, ue_bufIdx = read_data_from_shm( + shm_reading_ue, ue_bufIdx, tracer_msgs_identities) + + # Write data into files with the given format + if config_meta_data["data_recording_config"]["enable_saving_tracer_messages_sigmf"]: + collected_metafiles.append( + sigmf_interface.write_recorded_data_to_sigmf( + captured_data, config_meta_data, global_info, record_idx)) + + # generate SigMF collection file + if config_meta_data["data_recording_config"]["enable_saving_tracer_messages_sigmf"]: + data_storage_path = config_meta_data["data_recording_config"][ + "data_storage_path"] + description = global_info["description"] + sigmf_interface.save_sigmf_collection( + collected_metafiles, global_info, description, data_storage_path) + + frame = captured_data["frame"] + slot = captured_data["slot"] + + # Check for changes in frame or slot + if frame != prev_frame or slot != prev_slot: + record_idx += 1 + # We have reached the end of the data. Break the loop + if record_idx >= config_meta_data["data_recording_config"]["num_records"]: + break + # Update previous frame and slot + prev_frame = frame + prev_slot = slot + + +# data conversion service +def data_conversion_service(shm_reading, config_meta_data, args, sync_info, do_sync): + # Initialize variables + record_idx = 0 + prev_frame = -1 + prev_slot = -1 + bufIdx = 0 + # Read data from T-tracer Application + print("Data Conversion Service: Reading data from T-tracer Application") + print("Requested Tracer Messages: ", args.num_requested_tracer_msgs) + if args.num_requested_tracer_msgs > 0: + num_requested_tracer_msgs = args.num_requested_tracer_msgs + else: + raise Exception("ERROR: No requested tracer messages found!") + + tracer_msgs_identities = config_meta_data["data_recording_config"][ + "tracer_msgs_identities"] + global_info = config_meta_data["data_recording_config"]["global_info"] + + if do_sync: + print(" Find Memory Index of NR MSGs based on Sync info") + while True: + time.sleep(0.0035) + station_bufIdx = bufIdx + captured_data, bufIdx = read_data_from_shm( + shm_reading, bufIdx, tracer_msgs_identities) + print("*Sync Status: NR Captured Data (Frame, slot): (", captured_data["frame"], + ", ", captured_data["slot"], "), Sync Info (Frame, slot): (", sync_info["frame"], + ", ", sync_info["slot"]," )") + if (captured_data["frame"] == sync_info["frame"] + and captured_data["slot"] == sync_info["slot"]): + print("*sync Pass") + break + print("*sync Fail") + bufIdx = station_bufIdx + + # Read data from T-tracer Application + while True: # read all records + print("\nRecord number: ", record_idx) + if DEBUG_BUFFER_READING: + print(f"Buffer Index: {bufIdx}") + # wait for the next record + # To do: check if we need to add exta waiting times between different events in case of + # data streaming via network such as on UE side or gNB side + time.sleep(0.0035) # 2.3 ms = latency of T tracer to capture data from the RAN + + collected_metafiles = [] + for idx in range(num_requested_tracer_msgs): + time.sleep(0.0015) + if DEBUG_WIRELESS_RECORDED_DATA: + print(f"\nRecord number: {record_idx}, Reading MSG data ", idx) + captured_data, bufIdx = read_data_from_shm( + shm_reading, bufIdx, tracer_msgs_identities) + # derive the collection file time stamp from the first message per record + if idx == 0: + # Get time stamp + time_stamp_ms, time_stamp_ms_file_name = ( + sigmf_interface.time_stamp_formating( + captured_data["time_stamp"], global_info["datetime_offset"])) + global_info["collection_file"] = ( + global_info["collection_file_prefix"] + + "-rec-" + + str(record_idx) + + "-" + + str(time_stamp_ms_file_name)) + global_info["timestamp"] = time_stamp_ms + # Write data into files with the given format + if config_meta_data["data_recording_config"]["enable_saving_tracer_messages_sigmf"]: + collected_metafiles.append( + sigmf_interface.write_recorded_data_to_sigmf( + captured_data, config_meta_data, global_info, record_idx)) + + frame = captured_data["frame"] + slot = captured_data["slot"] + + # generate SigMF collection file + if config_meta_data["data_recording_config"]["enable_saving_tracer_messages_sigmf"]: + data_storage_path = config_meta_data["data_recording_config"][ + "data_storage_path"] + description = global_info["description"] + sigmf_interface.save_sigmf_collection( + collected_metafiles, global_info, description, data_storage_path) + + # Check for changes in frame or slot + if frame != prev_frame or slot != prev_slot: + record_idx += 1 + # We have reached the end of the data. Break the loop + if record_idx >= config_meta_data["data_recording_config"]["num_records"]: + break + # Update previous frame and slot + prev_frame = frame + prev_slot = slot + + +# Write Tracer Control Message +def write_shm(shm, args): + # Note: Big Endian >, Little Endian < + # Note: unsigned char B, signed char b, short h, int I, long long q + # Note: float f, double d, string s, char c, bool ? + # 1: config + # 2: record + # 3: quit + print("Write Shared Memory: ", args.action) + if args.action == "config": + # Determine the length of the IP address + ip_length = len(args.bytes_IPaddress) + 1 # String terminator + # Construct the format string dynamically + format_string = f"{ip_length}s" + shm.write( + # Config action + struct.pack(" dict: + """ + Reads main config file. + """ + # read general parameter set from yaml config file + with open(main_config_file, "r") as file: + main_config = json.load(file) + + return main_config + + +# Function to read the main configuration file in YAML format +def read_main_config_file(main_config_file: str) -> dict: + """ + Reads main config file. + """ + # read general parameter set from yaml config file + with open(main_config_file, "r") as file: + main_config = yaml.load(file, Loader=yaml.Loader) + + return main_config + + +# Function to parse the OAI T_messages file and get the index of a given string +def parse_message_file(file_path): + with open(file_path, "r") as file: + content = file.readlines() + # Extract lines that start with 'ID' and remove the 'ID = ' prefix + tracer_msgs_identities = [ + line.strip().replace("ID = ", "") + for line in content + if line.strip().startswith("ID") + ] + + return tracer_msgs_identities + + +# Function to get the index of a given string in the list of ID lines +def get_index_of_id(tracer_msgs_identities, message_id): + try: + return tracer_msgs_identities.index(message_id) + except ValueError: + return -1 # Return -1 if the string is not found + + +# Function to get the indices of requested tracer messages +def get_requested_tracer_msgs_indices(requested_tracer_messages, tracer_msgs_identities): + # get requested tracer messages indices + req_tracer_msgs_indices = [] + for idx, value in enumerate(requested_tracer_messages): + msg_index = get_index_of_id(tracer_msgs_identities, value) + req_tracer_msgs_indices.append(msg_index) + print("Requested Traces IDs: ", req_tracer_msgs_indices) + return req_tracer_msgs_indices + + +# Function to get the data recording configuration +def get_data_recording_config(config_meta_data): + parser = argparse.ArgumentParser(description="request messages IDs") + ue_args = parser.parse_args() + gnb_args = parser.parse_args() + + # get Tracer Messages IDs from the T-Tracer Messages file + # MSG list order should be similar to the T-Tracer Messages txt file + # Be sure that you are using the same T_messages file that is used in the OAI project + # OAI Path: .../common/utils/T/T_messages.txt + + # Parse the T_messages file + tracer_msgs_identities = parse_message_file( + config_meta_data["data_recording_config"]["t_tracer_message_definition_file"]) + config_meta_data["data_recording_config"]["tracer_msgs_identities"] = tracer_msgs_identities + + # get requested tracer messages indices for gNB + gnb_args.requested_tracer_messages = \ + config_meta_data["data_recording_config"]["base_station"]["requested_tracer_messages"] + # get requested tracer messages indices for UE + ue_args.requested_tracer_messages = \ + config_meta_data["data_recording_config"]["user_equipment"]["requested_tracer_messages"] + + # check if gnb_requested_tracer_messages is not empty + if gnb_args.requested_tracer_messages: + config_meta_data["data_recording_config"]["base_station"][ + "req_tracer_msgs_indices"] = get_requested_tracer_msgs_indices( + gnb_args.requested_tracer_messages, tracer_msgs_identities) + + # get gNB Trace Messages + gnb_args.num_records = config_meta_data["data_recording_config"]["num_records"] + gnb_args.start_frame_number = config_meta_data["data_recording_config"]["start_frame_number"] + gnb_args.req_tracer_msgs_indices = \ + config_meta_data["data_recording_config"]["base_station"]["req_tracer_msgs_indices"] + gnb_args.num_requested_tracer_msgs = len(gnb_args.req_tracer_msgs_indices) + # Split the string into IP and port + gnb_args.IPaddress, gnb_args.port = config_meta_data["data_recording_config"][ + "tracer_service_baseStation_address"].split(":") + gnb_args.bytes_IPaddress = bytes(gnb_args.IPaddress, "utf-8") + + # check if ue_requested_tracer_messages is not empty + if ue_args.requested_tracer_messages: + config_meta_data["data_recording_config"]["user_equipment"][ + "req_tracer_msgs_indices"] = get_requested_tracer_msgs_indices( + ue_args.requested_tracer_messages, tracer_msgs_identities) + + # get UE Trace Messages + ue_args.num_records = config_meta_data["data_recording_config"]["num_records"] + ue_args.start_frame_number = config_meta_data["data_recording_config"]["start_frame_number"] + ue_args.req_tracer_msgs_indices = \ + config_meta_data["data_recording_config"]["user_equipment"]["req_tracer_msgs_indices"] + ue_args.num_requested_tracer_msgs = len(ue_args.req_tracer_msgs_indices) + ue_args.IPaddress, ue_args.port = config_meta_data["data_recording_config"][ + "tracer_service_userEquipment_address"].split(":") + ue_args.bytes_IPaddress = bytes(ue_args.IPaddress, "utf-8") + + return config_meta_data, gnb_args, ue_args diff --git a/common/utils/data_recording/lib/data_recording_messages_def.py b/common/utils/data_recording/lib/data_recording_messages_def.py new file mode 100644 index 0000000000..a807058f29 --- /dev/null +++ b/common/utils/data_recording/lib/data_recording_messages_def.py @@ -0,0 +1,107 @@ +# /* +# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more +# * contributor license agreements. See the NOTICE file distributed with +# * this work for additional information regarding copyright ownership. +# * The OpenAirInterface Software Alliance licenses this file to You under +# * the OAI Public License, Version 1.1 (the "License"); you may not use this file +# * except in compliance with the License. +# * You may obtain a copy of the License at +# * +# * http://www.openairinterface.org/?page_id=698 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# *------------------------------------------------------------------------------- +# * For more information about the OpenAirInterface (OAI) Software Alliance: +# * contact@openairinterface.org +# */ +# --------------------------------------------------------------------- +# file common/utils/data_recording/lib/data_recording_messages_def.py +# brief defination of captured data recording messages +# author Abdo Gaber +# date 2025 +# version 1.0 +# company Emerson, NI Test and Measurement +# email: +# note +# warning + + +# Data Collection Trace Messages - General message structure - number of bytes +def get_general_msg_header_list(): + """ + shared memory layout written from the app: + ================================= + msg_id (uint8) message type ID + frame (uint16) + slot (uint8) + datetime_yyyymmdd (uint32) + datetime_hhmmssmmm (uint32) + frame_type (uint8) + freq_range (uint8) + subcarrier_spacing (uint8) + cyclic_prefix (uint8) + symbols_per_slot (uint8) + Nid_cell (uint16) + rnti (uint16) + rb_size (uint16) + rb_start (uint16) + start_symbol_index (uint8) + nr_of_symbols (uint8) + qam_mod_order (uint8) + mcs_index (uint8) + mcs_table (uint8) + nrOfLayers (uint8) + transform_precoding (uint8) + dmrs_config_type (uint8) + ul_dmrs_symb_pos (uint16) + number_dmrs_symbols (uint8) + dmrs_port (uint16) + dmrs_scid (uint16) + nb_antennas (uint8) + number_of_bits (uint32) + length_bytes (uint32) + For IQ Data: IQ samples: I0, Q0, I1, Q1, ... I_x, Q_x (int16) + For bit data: bits: b0, b1, b2, ... b_x (uint8) + + """ + # Data Collection Trace Messages - General message structure - number of bytes + general_msg_header_list = { + "msg_id": 2, + "frame": 2, + "slot": 1, + "datetime_yyyymmdd": 4, + "datetime_hhmmssmmm": 4, + "frame_type": 1, + "freq_range": 1, + "subcarrier_spacing": 1, + "cyclic_prefix": 1, + "symbols_per_slot": 1, + "Nid_cell": 2, + "rnti": 2, + "rb_size": 2, + "rb_start": 2, + "start_symbol_index": 1, + "nr_of_symbols": 1, + "qam_mod_order": 1, + "mcs_index": 1, + "mcs_table": 1, + "nrOfLayers": 1, + "transform_precoding": 1, + "dmrs_config_type": 1, + "ul_dmrs_symb_pos": 2, + "number_dmrs_symbols": 1, + "dmrs_port": 2, + "dmrs_scid": 2, + "nb_antennas": 1, # for gNB or nb_antennas_tx for UE + "number_of_bits": 4, + "length_bytes": 4, + } + # initial number of bytes to read to get data + general_message_header_length = 0 + for key, value in general_msg_header_list.items(): + general_message_header_length = general_message_header_length + value + return general_msg_header_list, general_message_header_length diff --git a/common/utils/data_recording/lib/sigmf_interface.py b/common/utils/data_recording/lib/sigmf_interface.py new file mode 100644 index 0000000000..74d58a03f8 --- /dev/null +++ b/common/utils/data_recording/lib/sigmf_interface.py @@ -0,0 +1,580 @@ +# /* +# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more +# * contributor license agreements. See the NOTICE file distributed with +# * this work for additional information regarding copyright ownership. +# * The OpenAirInterface Software Alliance licenses this file to You under +# * the OAI Public License, Version 1.1 (the "License"); you may not use this file +# * except in compliance with the License. +# * You may obtain a copy of the License at +# * +# * http://www.openairinterface.org/?page_id=698 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# *------------------------------------------------------------------------------- +# * For more information about the OpenAirInterface (OAI) Software Alliance: +# * contact@openairinterface.org +# */ +# --------------------------------------------------------------------- +# file common/utils/data_recording/lib/sigmf_interface.py +# brief SigMF Interface to Write data and meta-data to files in SigMF format +# author Abdo Gaber +# date 2025 +# version 1.0 +# company Emerson, NI Test and Measurement +# email: +# note +# warning + +import os +import sigmf +from sigmf import SigMFFile +# from sigmf.utils import get_data_type_str +import numpy as np +from datetime import datetime +import yaml +from sigmf import SigMFCollection + +""" +SERIALIZATION_SCHEMES = { + # gnb + "rx-fd-data": ["subcarriers", "ofdm_symbols"], + "tx-pilots-fd-data": ["subcarriers", "ofdm_symbols"], + "raw-ce-fd-data": ["subcarriers", "ofdm_symbols"], + "rx-payload-bits": ["bits", "subcarriers", "ofdm_symbols"], + # ue + "tx-scrambled-bits": ["bits", "subcarriers", "ofdm_symbols"], + "tx-payload-bits": ["bits", "subcarriers", "ofdm_symbols"], +} +""" +STANDARDS = {"5gnr_oai": "5gnr"} + + +def time_stamp_formating(time_stamp, datetime_offset): + # Parse the input string into a datetime object + time_stamp_ms_obj = datetime.strptime(time_stamp, "%Y%m%d_%H%M%S%f") + + # Format the datetime object into the desired output format with milliseconds + time_stamp_ms_iso = ( + time_stamp_ms_obj.strftime("%Y_%m_%dT%H:%M:%S.%f")[:-3] + datetime_offset + ) + time_stamp_ms = time_stamp_ms_iso + time_stamp_ms_file_name = time_stamp_ms_iso.replace(":", "_").replace(".", "_") + + return time_stamp_ms, time_stamp_ms_file_name + + +def create_serialization_metadata(serialization_scheme, data_source: str, + link_sim_parameters: dict) -> dict: + """Creates dict that specifies the serialization metadata.""" + + # retrieve parameter from LinkSimulator config + if data_source == "5gnr_oai": + num_ofdm_symbol = link_sim_parameters["nr_of_symbols"] + num_subcarriers = link_sim_parameters["rb_size"] * 12 + num_bits_per_symbol = link_sim_parameters["qam_mod_order"] + else: + raise Exception( + f"Invalid data source string '{data_source}'! Only '5gnr_oai' is valid!" + ) + + # create array with scheme dimensions + if len(serialization_scheme) == 2: + counts = [num_subcarriers, num_ofdm_symbol] + elif len(serialization_scheme) == 3: + counts = [num_bits_per_symbol, num_subcarriers, num_ofdm_symbol] + else: + raise Exception( + f"Invalid number of elements in serialization scheme: {len(serialization_scheme)}") + + # create metadata dict + serialization_dict = { + "serialization:scheme": serialization_scheme, + "serialization:counts": counts, + } + + return serialization_dict + + +def map_metadata_to_sigmf_format(scope, waveform_generator, parameter_map_file, captured_data): + """ + Maps metadata from Waveform creator to API and SigMF format. + The used parameters and the mapping pairs are specified in a separate YAML file + that must be provided as well. + """ + # read waveform parameter map from yaml file + dir_path = os.path.dirname(__file__) + src_path = os.path.split(dir_path)[0] + + with open(os.path.join(src_path, parameter_map_file), "r") as file: + parameter_map_dic = yaml.load(file, Loader=yaml.Loader) + # preallocate target dict + sigmf_metadata_dict = {} + + # get standard and name of generator + standard_key = parameter_map_dic["waveform_generator"][waveform_generator] + generator = standard_key["generator"] + + if scope == "tx": + parameter_map_dic = parameter_map_dic["transmitter"][ + STANDARDS[waveform_generator] + ] + elif scope == "channel": + parameter_map_dic = parameter_map_dic["channel"] + elif scope == "rx": + parameter_map_dic = parameter_map_dic["receiver"] + else: + raise Exception( + f"Invalid mapping scope '{scope}'! Only 'tx', 'channel' and 'rx' are valid!" + ) + + # check if standard key is given + if parameter_map_dic is None: + raise Exception( + "Invalid standard key: " + "Name should be corrected or added to wireless_link_parameter_map.yaml, given: " + f"{waveform_generator}" + ) + + for parameter_pair in parameter_map_dic: + # check if key for chosen simulator even exists + if waveform_generator + "_parameter" in parameter_pair.keys(): + # only continue with mapping from file if direct equivalent exists + if parameter_pair[waveform_generator + "_parameter"]["name"]: + # It is not necessary to get all parameters from wireless_link_parameter_map.yaml + # in captured_data since some parameters related to DL or UL only + if (parameter_pair[waveform_generator + "_parameter"]["name"] in captured_data.keys()): + # extract value from waveform config source + value = captured_data[ + parameter_pair[waveform_generator + "_parameter"]["name"]] + # additional mapping if parameter values should come from a discrete set of values + if ("value_map" in parameter_pair[waveform_generator + "_parameter"].keys()): + value = parameter_pair[waveform_generator + "_parameter"]["value_map"][value] + # write to target dictionary for SigMF + sigmf_metadata_dict[parameter_pair["sigmf_parameter_name"]] = value + else: + raise Exception( + f"Incomplete specification in field '{waveform_generator}_parameter'!") + # else: # fill_non_explicit_fields + # waveform_config[parameter_pair["sigmf_parameter_name"]] = "none" + if not sigmf_metadata_dict: + raise Exception( + """ERROR: Check captured meta-data or provided config and meta data """) + + # check for non-JSON-serializable data types + def isfloat(NumberString): + try: + float(NumberString) + return True + except ValueError: + return False + + for key, value in sigmf_metadata_dict.items(): + if isinstance(value, np.integer): + sigmf_metadata_dict[key] = int(value) + elif isinstance(value, (np.float16, np.float32, np.float64)): + sigmf_metadata_dict[key] = np.format_float_positional(value, trim="-") + elif isinstance(value, int): + sigmf_metadata_dict[key] = int(value) + elif isinstance(value, float): + # store value in decimal and not in scientific notation + sigmf_metadata_dict[key] = float(value) + elif isinstance(value, str) and key != "standard": + # convert string to lower case + sigmf_metadata_dict[key] = value.lower() + if isfloat(value): + if value.isdigit(): + sigmf_metadata_dict[key] = int(float(value)) + elif value.replace(".", "", 1).isdigit() and value.count(".") < 2: + sigmf_metadata_dict[key] = float(value) + + return sigmf_metadata_dict, generator + + +def create_system_components_metadata(waveform_generator, parameter_map_file, captured_data): + """Creates system components (TX, channel, RX) metadata that will reside in the annotations.""" + # map metadata of waveform generator to SigMF format + signal_info, generator = map_metadata_to_sigmf_format( + "tx", waveform_generator, parameter_map_file, captured_data + ) + tx_metadata = { + "signal:detail": { + "standard": STANDARDS[waveform_generator], + "generator": generator, + STANDARDS[waveform_generator]: signal_info, + } + } + + channel_metadata = {} # will be filled later + rx_metadata = {} # will be filled later + + return tx_metadata, channel_metadata, rx_metadata + + +def write_recorded_data_to_sigmf(captured_data, config_meta_data, global_info, idx): + """ + Compiles and saves provided data and metadata into SigMF file format. + """ + # get meta data from config file + base_station_meta_data = config_meta_data["data_recording_config"]["base_station"][ + "meta_data"] + user_equipment_meta_data = config_meta_data["data_recording_config"][ + "user_equipment"]["meta_data"] + + # Check the receive target path is valid, else create folder + data_storage_path = config_meta_data["data_recording_config"]["data_storage_path"] + if not os.path.isdir(data_storage_path): + print("Create new folder for recorded data: " + str(data_storage_path)) + os.makedirs(data_storage_path) + + # Write recorded data to file + # Get time stamp + time_stamp_ms, time_stamp_ms_file_name = time_stamp_formating( + captured_data["time_stamp"], global_info["datetime_offset"]) + # Map OAI Message Name to SigMF Message Name + file_name_prefix = config_meta_data["data_recording_config"][ + "supported_oai_tracer_messages"][captured_data["message_type"]]["file_name_prefix"] + recorded_data_file_name = ( + file_name_prefix + "-rec-" + str(idx) + "-" + time_stamp_ms_file_name) + + dataset_filename = recorded_data_file_name + ".sigmf-data" + dataset_file_path = os.path.join(data_storage_path, dataset_filename) + print(dataset_file_path) + captured_data["recorded_data"].tofile(dataset_file_path) + + # map OAI config data to SigMF metadata + waveform_generator = config_meta_data["data_recording_config"]["global_info"][ + "waveform_generator"] + parameter_map_file = config_meta_data["data_recording_config"]["parameter_map_file"] + tx_metadata, channel_metadata, rx_metadata = create_system_components_metadata( + waveform_generator, parameter_map_file, captured_data) + + # Add other OAI metadata to SigMF metadata + # ---------------------------------------------------- + # Set 1: Parameters needs to be derived from OAI message + # ---------------------------------------------------- + # PUSCH DMRS: OFDM symbol start index within slot + get_pusch_dmrs_start_ofdm_symbol = 0 + tx_metadata["signal:detail"][STANDARDS[waveform_generator]][ + "pusch_dmrs:ofdm_symbol_idx"] = [] + + for symbol in range(captured_data["start_symbol_index"], + captured_data["start_symbol_index"] + captured_data["nr_of_symbols"]): + + dmrs_symbol_flag = (captured_data["ul_dmrs_symb_pos"] >> symbol) & 0x01 + if dmrs_symbol_flag and not get_pusch_dmrs_start_ofdm_symbol: + get_pusch_dmrs_start_ofdm_symbol = 1 + tx_metadata["signal:detail"][STANDARDS[waveform_generator]][ + "pusch_dmrs:start_ofdm_symbol"] = symbol + if dmrs_symbol_flag: + tx_metadata["signal:detail"][STANDARDS[waveform_generator]][ + "pusch_dmrs:ofdm_symbol_idx"].append(symbol) # Append symbol to the list + + tx_metadata["signal:detail"][STANDARDS[waveform_generator]][ + "pusch_dmrs:duration_num_ofdm_symbols"] = 1 + tx_metadata["signal:detail"][STANDARDS[waveform_generator]][ + "pusch_dmrs:num_add_positions"] = (captured_data["number_dmrs_symbols"] - 1) + + # If the message is a BIT message, add number of bits to the signal info + # "UE_PHY_UL_SCRAMBLED_TX_BITS", "GNB_PHY_UL_PAYLOAD_RX_BITS", "UE_PHY_UL_PAYLOAD_TX_BITS" + if "_BITS" in captured_data["message_type"]: + # if "BIT" in captured_data["message_type"]: + tx_metadata["signal:detail"][STANDARDS[waveform_generator]]["num_bits"] = ( + captured_data["number_of_bits"]) + + # If the message is captured from UE, it is UPLink message, + # so number of antennas is num of Tx antennas + # number of antennas on Rx side should be read from global_info given by user + if "UE" in captured_data["message_type"]: + tx_metadata["signal:detail"][STANDARDS[waveform_generator]][ + "num_tx_antennas" + ] = captured_data["nb_antennas"] + rx_metadata.update( + {"num_rx_antennas": base_station_meta_data["num_rx_antennas"]} + ) + # If the message is captured from gNB, it is UPLink message, + # so number of antennas is num of Rx antennas + # number of antennas on Tx side should be read from global_info given by user + else: + tx_metadata["signal:detail"][STANDARDS[waveform_generator]][ + "num_tx_antennas"] = user_equipment_meta_data["num_tx_antennas"] + rx_metadata.update({"num_rx_antennas": captured_data["nb_antennas"]}) + + # Check if the message type contains "UL" + if "UL" in captured_data["message_type"]: + tx_metadata["signal:detail"][STANDARDS[waveform_generator]][ + "link_direction"] = "uplink" + else: + tx_metadata["signal:detail"][STANDARDS[waveform_generator]][ + "link_direction"] = "downlink" + # ---------------------------------------------------- + # Set 2: Parameters that is hardcoded + # ---------------------------------------------------- + tx_metadata["signal:detail"][STANDARDS[waveform_generator]][ + "pusch:content"] = "compliant" + tx_metadata["signal:detail"][STANDARDS[waveform_generator]][ + "pusch:mapping_type"] = "B" + tx_metadata["signal:detail"][STANDARDS[waveform_generator]]["num_slots"] = 1 + if config_meta_data["test_config"]["test_mode"] == "rf_simulation": + tx_metadata["signal:detail"][STANDARDS[waveform_generator]][ + "pusch:payload_bit_pattern"] = "random" + elif config_meta_data["test_config"]["test_mode"] == "rf_real_time": + tx_metadata["signal:detail"][STANDARDS[waveform_generator]][ + "pusch:payload_bit_pattern"] = "zeros" + else: + raise Exception("ERROR: Test mode is not supported in SigMF formatting") + # ---------------------------------------------------- + # Read mean parameters + freq = config_meta_data["environment_emulation"]["target_link_config"][ + "wireless_channel"]["carrierFreqValueList_hz"][0] + bandwidth = config_meta_data["data_recording_config"]["common_meta_data"][ + "bandwidth"] + # used_signal_bandwidth = ( + # pow(10, 6) * config_meta_data["environment_emulation"]["target_link_config"]["ran_config"][ + # "uplink"]["ul_used_signal_BW_MHz"]) + sample_rate = config_meta_data["data_recording_config"]["common_meta_data"][ + "sample_rate"] + + # get signal emitter info + signal_emitter = { + "manufacturer": "NI", + "seid": user_equipment_meta_data["seid"], # Unique ID of the emitter + "hw": user_equipment_meta_data["hw_type"], + "hw_subtype": user_equipment_meta_data["hw_subtype"], + "frequency": freq, + "sample_rate": sample_rate, + "bandwidth": bandwidth, + "gain_tx": user_equipment_meta_data["tx_gain"], + "clock_reference": config_meta_data["data_recording_config"][ + "common_meta_data" + ]["clock_reference"], + } + + # ---------------------------------------------------- + # get channel metadata + # ---------------------------------------------------- + # get downlink channel info + # ch_downlink_config = config_meta_data["environment_emulation"]["target_link_config"]["wireless_channel"]["downlink"] + # ch_downlink ={"channel_model": ch_downlink_config["type"]} + + # ch_downlink_model = {"channel_model": "AWGN"} + # get uplink channel info + ch_uplink_config = config_meta_data["environment_emulation"]["target_link_config"][ + "wireless_channel" + ]["uplink"] + # Real Time RF Emulation + if config_meta_data["test_config"]["test_mode"] == "rf_real_time": + channel_metadata.update({"emulation_mode": "ni rf real time"}) + if ch_uplink_config["type"] == "statistical_predefined": + channel_metadata.update( + { + "channel_model": ch_uplink_config["statistical"]["predef_channel_profile"], + "snr_esn0_db": ch_uplink_config["statistical"]["snr_db"], + } + ) + + elif ch_uplink_config["type"] == "statistical_var": + channel_metadata.update( + { + "channel_model": ch_uplink_config["statistical"]["var_power_delay_profile"], + "snr_esn0_db": ch_uplink_config["statistical"]["snr_db"], + "delay_spread": ch_uplink_config["statistical"]["delay_spread_ns"] * pow(10, -9), + "speed": ch_uplink_config["statistical"]["speed_mps"], + } + ) + else: + raise Exception("ERROR: channel type is not supported in SigMF formatting") + # RF Simulation + elif config_meta_data["test_config"]["test_mode"] == "rf_simulation": + channel_metadata.update({"emulation_mode": "oai rf simulation"}) + channel_metadata.update( + { + "channel_model": ch_uplink_config["statistical"]["predef_channel_profile"], + "snr_esn0_db": ch_uplink_config["statistical"]["snr_db"], + "delay_spread": ch_uplink_config["statistical"]["delay_spread_ns"] + * pow(10, -9), "speed": ch_uplink_config["statistical"]["speed_mps"], + } + ) + else: + raise Exception( + "ERROR: Channel Emulation mode is not supported in SigMF formatting") + + channel_metadata["carrier_frequency"] = config_meta_data["environment_emulation"][ + "target_link_config"]["wireless_channel"]["carrierFreqValueList_hz"][0] + + # ---------------------------------------------------- + # get rx metadata + # ---------------------------------------------------- + rx_metadata.update( + { + "bandwidth": bandwidth, + "gain": base_station_meta_data["rx_gain"], + "manufacturer": "NI", + "seid": base_station_meta_data["seid"], + "hw_subtype": base_station_meta_data["hw_subtype"], + "clock_reference": config_meta_data["data_recording_config"][ + "common_meta_data" + ]["clock_reference"], + "phy_freq_domain_receiver_type": config_meta_data["test_config"][ + "dut_type" + ], + } + ) + # ---------------------------------------------------- + # get test config + # ---------------------------------------------------- + """ + test_config = { + "test_name": config_meta_data["test_config"]["test_name"], + "test_mode": config_meta_data["test_config"]["test_mode"], + "dut_type": config_meta_data["test_config"]["dut_type"], + } + """ + # Create sigmf metadata + # ---------------------- + # Add global parameters to SigMF metadata + # ---------------------- + meta = SigMFFile( + data_file=dataset_file_path, # extension is optional + global_info={ + SigMFFile.DATATYPE_KEY: captured_data[ + "sigmf_data_type" + ], # "cf32_le", # get_data_type_str(rx_data) - 'cf64_le' is not supported yet + SigMFFile.SAMPLE_RATE_KEY: sample_rate, + SigMFFile.NUM_CHANNELS_KEY: 1, + SigMFFile.AUTHOR_KEY: global_info["author"], + SigMFFile.DESCRIPTION_KEY: config_meta_data["data_recording_config"][ + "supported_oai_tracer_messages" + ][captured_data["message_type"]]["description"], + SigMFFile.RECORDER_KEY: "NI Data Recording Application for OAI", + SigMFFile.LICENSE_KEY: "MIT License", + # Since we are focusing on 5G NR UL, the base station is the receiver + SigMFFile.HW_KEY: base_station_meta_data["hw_type"], + # Disable DATASET key to mitigate the warning when read SIGMF data although it is given in the spec. + # It seems SIGMF still has bug here + SigMFFile.DATASET_KEY: dataset_filename, + SigMFFile.VERSION_KEY: sigmf.__version__, + sigmf.SigMFFile.COLLECTION_KEY: global_info["collection_file"], + }, + ) + + # ---------------------- + # Add capture parameters to SigMF metadata + # ---------------------- + serialization_scheme = config_meta_data["data_recording_config"][ + "supported_oai_tracer_messages" + ][captured_data["message_type"]]["serialization_scheme"] + capture_metadata = { + sigmf.SigMFFile.DATETIME_KEY: time_stamp_ms, + SigMFFile.FREQUENCY_KEY: freq, + **create_serialization_metadata( + serialization_scheme, waveform_generator, captured_data + ), + } + meta.add_capture(start_index=0, metadata=capture_metadata) # Sample Start + + # create annotations dict + system_components_dict = { + "system_components:transmitter": [ + { + "transmitter_id": "tx_0", + **tx_metadata, + "signal:emitter": signal_emitter, + } + ] + } + + system_components_dict["system_components:channel"] = [ + {"transmitter_id": "tx_0", **channel_metadata}] + + system_components_dict["system_components:receiver"] = rx_metadata + # ---------------------- + # Add annotation parameters to SigMF metadata + # ---------------------- + meta.add_annotation( + start_index=0, # Sample Start + length=len(captured_data["recorded_data"]), # Sample count + metadata={ + # SigMFFile.FLO_KEY: freq - sample_rate/ 2, # args.freq - args.rate / 2, + # SigMFFile.FHI_KEY: freq + sample_rate/ 2, # args.freq + args.rate / 2, + # SigMFFile.LABEL_KEY: label, + # SigMFFile.COMMENT_KEY: general_config["comment"], + "num_transmitters": 1, + **system_components_dict, + }, + ) + + # Write Meta Data to file + dataset_meta_filename = recorded_data_file_name + ".sigmf-meta" + dataset_meta_file_path = os.path.join(data_storage_path, dataset_meta_filename) + meta.tofile(dataset_meta_file_path) # extension is optional + + print(dataset_meta_file_path) + + return dataset_meta_file_path + + +def save_sigmf_collection(streams: list, global_info: dict, description: str, storage_path: str): + """Save metadata and links to SigMF data/metadata files to SigMF collection file.""" + # construct path + storage_path = os.path.expanduser(storage_path) + collection_filepath = os.path.join(storage_path, global_info["collection_file"]) + # statically configure Collection metadata + collection = SigMFCollection( + streams, + metadata={ + SigMFCollection.COLLECTION_KEY: { + SigMFCollection.VERSION_KEY: sigmf.__version__, + SigMFCollection.DESCRIPTION_KEY: description, + SigMFCollection.AUTHOR_KEY: global_info["author"], + # SigMFCollection.LICENSE_KEY: "License", + # SigMFCollection.EXTENSIONS_KEY: global_info["extensions"], + SigMFCollection.STREAMS_KEY: [], + } + }, + ) + + # save collection file + collection.tofile(collection_filepath) + print("") + print(collection_filepath + ".sigmf-collection") + + +def load_sigmf(filename: str, storage_path: str, scope: str): + """ + Loads metadata and data from SigMF file. + """ + # construct path + storage_path = os.path.expanduser(storage_path) + filepath = os.path.join(storage_path, filename) + # load SigMF file + signal = sigmf.sigmffile.fromfile(filepath) + + # get metadata fields + global_metadata = signal.get_global_info() + annotations_metadata = signal.get_annotations() + + # we only consider single annotations metadata element per SigMF file + annotation_start_idx = annotations_metadata[0][sigmf.SigMFFile.START_INDEX_KEY] + annotation_length = annotations_metadata[0][sigmf.SigMFFile.LENGTH_INDEX_KEY] + + # get capture metadata + capture_metadata = signal.get_capture_info(annotation_start_idx) + + # from source code: sigmffile.py + # "autoscale : bool, default True + # If dataset is in a fixed-point representation, scale samples from (min, max) to (-1.0, 1.0) + # raw_components : bool, default False + # If True read and return the sample components (individual I & Q for complex, samples for real) + # with no conversions or interleaved channels."" + # TODO: problem: raw_components flag is not used at all! changes in source code required! + # TODO: data is always converted to float32 after being read + + # read actual data + data = signal.read_samples( + annotation_start_idx, annotation_length, autoscale=False, raw_components=True) + + return data, global_metadata, annotations_metadata diff --git a/common/utils/data_recording/lib/sync_service.py b/common/utils/data_recording/lib/sync_service.py new file mode 100644 index 0000000000..87538173fd --- /dev/null +++ b/common/utils/data_recording/lib/sync_service.py @@ -0,0 +1,150 @@ + +# /* +# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more +# * contributor license agreements. See the NOTICE file distributed with +# * this work for additional information regarding copyright ownership. +# * The OpenAirInterface Software Alliance licenses this file to You under +# * the OAI Public License, Version 1.1 (the "License"); you may not use this file +# * except in compliance with the License. +# * You may obtain a copy of the License at +# * +# * http://www.openairinterface.org/?page_id=698 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# *------------------------------------------------------------------------------- +# * For more information about the OpenAirInterface (OAI) Software Alliance: +# * contact@openairinterface.org +# */ +# --------------------------------------------------------------------- +# file common/utils/data_recording/lib/sync_service.py +# brief Sync service between captured Data from 5GNR gNB and UE +# author Abdo Gaber +# date 2025 +# version 1.0 +# company Emerson, NI Test and Measurement +# email: +# note +# warning + +import struct +from lib import data_recording_messages_def + +DEBUG_POW_MEAS_SYNC = True + + +# check if first frame ahead: +def is_frame_ahead(frame1, frame2, max_frame=1023): + """ + Check if frame1 is ahead of frame2, considering wrap-around from max_frame to 0. + + Args: + frame1 (int): The first frame number. + frame2 (int): The second frame number. + max_frame (int): The maximum frame number before wrap-around. Default is 1023. + + Returns: + bool: True if frame1 is ahead of frame2, False otherwise. + """ + # Calculate the difference considering wrap-around + diff = (frame1 - frame2 + (max_frame + 1)) % (max_frame + 1) + # If the difference is less than half the range, frame1 is ahead + return diff < (max_frame + 1) // 2 + + +# find the frame and slot start +def find_frame_slot_start(dataset1_start, dataset2_start): + """ + Function to find the frame and slot start for data sync + Args: + dataset1_start_info: Dictionary containing the start information of dataset1 + dataset2_start_info: Dictionary containing the start information of dataset2 + Returns: + sync_info: Dictionary containing the sync information + sync_info["frame"] = frame_start + sync_info["slot"] = slot_start + + """ + sync_info = {} + + if dataset1_start["frame"] == dataset2_start["frame"]: + frame_diff = 0 + frame_start = dataset1_start["frame"] + slot_start = max(dataset1_start["slot"], dataset2_start["slot"]) + + elif is_frame_ahead(dataset1_start["frame"], dataset2_start["frame"]): + frame_start = dataset1_start["frame"] + slot_start = dataset1_start["slot"] + frame_diff = (dataset1_start["frame"]- dataset2_start["frame"] + 1024) % 1024 + elif is_frame_ahead(dataset2_start["frame"], dataset1_start["frame"]): + frame_start = dataset2_start["frame"] + slot_start = dataset2_start["slot"] + frame_diff = (dataset2_start["frame"] - dataset1_start["frame"] + 1024) % 1024 + # check first if the delta time between the two datasets is larger than expected offset time. + # So, the sync will not be applied + # check during the calculation of the frame the ramp-around from 1023 to 0 + if abs(frame_diff) > 6: + raise Exception( + f"Frame difference between dataset1 and dataset2 is too large: {frame_diff}. " + "This may indicate a problem with the data collection or synchronization." + ) + + # Determine the starting frame and slot for data sync + sync_info["frame"] = frame_start + sync_info["slot"] = slot_start + return sync_info + + +# Read data from gNB T-tracer Application +def get_frame_slot_start(shm_reading, bufIdx, general_msg_header_list, + general_message_header_length): + buf = shm_reading.read(bufIdx + general_message_header_length) + msg_id = struct.unpack("