--- title: Lidars and Cameras Sequence slug: api-guide/lidars-and-cameras-sequence docTags: createdAt: 2024-12-16T14:19:04.063Z --- :::hint{type="info"} Since kognic-io 2.5.0 we are promoting our newer scene model, which does not expose scene types (`LidarsAndCamerasSequence`) and simplifies creation of any scene from a single model. ::: A `LidarsAndCamerasSequence` consists of a sequence of camera images and lidar point clouds, where each frame consists on 1-20 camera images as well as 1-20 point clouds. For more documentation on what each field corresponds to in the `LidarsAndCamerasSequence` object please check the section related to [Scene Overview](docId\:yUnPNPwUhZlGG9Wb9Qnk8). ```python from __future__ import absolute_import from datetime import datetime from typing import Optional from uuid import uuid4 import kognic.io.model.scene.lidars_and_cameras_sequence as LCSM from examples.calibration.calibration import create_sensor_calibration from kognic.io.client import KognicIOClient from kognic.io.logger import setup_logging from kognic.io.model import CreateSceneResponse, Image, PointCloud def run(client: KognicIOClient, dryrun: bool = True, **kwargs) -> Optional[CreateSceneResponse]: print("Creating Lidar and Camera Sequence Scene...") lidar_sensor1 = "lidar" cam_sensor1 = "RFC01" cam_sensor2 = "RFC02" metadata = {"location-lat": 27.986065, "location-long": 86.922623, "vehicle_id": "abg"} # Create calibration # (Please refer to the API documentation about calibration for more details) calibration_spec = create_sensor_calibration( f"Collection {datetime.now()}", [lidar_sensor1], [cam_sensor1, cam_sensor2], ) created_calibration = client.calibration.create_calibration(calibration_spec) scene = LCSM.LidarsAndCamerasSequence( external_id=f"LCS-example-{uuid4()}", frames=[ LCSM.Frame( frame_id="1", relative_timestamp=0, point_clouds=[ PointCloud( filename="./examples/resources/point_cloud_RFL01.las", sensor_name=lidar_sensor1, ), ], images=[ Image( filename="./examples/resources/img_RFC01.jpg", sensor_name=cam_sensor1, ), Image( filename="./examples/resources/img_RFC02.jpg", sensor_name=cam_sensor2, ), ], metadata={"dut_status": "active"}, ), LCSM.Frame( frame_id="2", relative_timestamp=100, point_clouds=[ PointCloud( filename="./examples/resources/point_cloud_RFL02.las", sensor_name=lidar_sensor1, ), ], images=[ Image( filename="./examples/resources/img_RFC11.jpg", sensor_name=cam_sensor1, ), Image( filename="./examples/resources/img_RFC12.jpg", sensor_name=cam_sensor2, ), ], metadata={"dut_status": "active"}, ), ], calibration_id=created_calibration.id, metadata=metadata, ) # Create scene return client.lidars_and_cameras_sequence.create(scene, dryrun=dryrun, **kwargs) if __name__ == "__main__": setup_logging(level="INFO") client = KognicIOClient() # Project - Available via `client.project.get_projects()` project = "" run(client, project) ``` :::hint{type="success"} ### Use dryrun to validate scene Setting `dryrun` parameter to true in the method call, will validate the scene using the API but not create it. ::: :::hint{type="success"} ### Reuse calibration Note that you can, and should, reuse the same calibration for multiple scenes if possible. ::: ## Providing Ego Vehicle Motion Information Ego vehicle motion (i.e. the position and rotation of the ego vehicle) is optional information that can be provided when creating `LidarsAndCamerasSeq`s. This information can enable a massive reduction in the time it takes to annotate static objects. Ego vehicle motion information is provided by passing a `EgoVehicleMotion` object to **each** `Frame`in the scene. ```python from __future__ import absolute_import from datetime import datetime from typing import Optional from uuid import uuid4 import kognic.io.model.scene.lidars_and_cameras_sequence as LCSM from examples.calibration.calibration import create_sensor_calibration from kognic.io.client import KognicIOClient from kognic.io.logger import setup_logging from kognic.io.model import CreateSceneResponse, EgoVehiclePose, Image, PointCloud, Position, RotationQuaternion def run(client: KognicIOClient, dryrun: bool = True, **kwargs) -> Optional[CreateSceneResponse]: print("Creating Lidar and Camera Sequence Scene...") lidar_sensor1 = "lidar" cam_sensor1 = "RFC01" cam_sensor2 = "RFC02" cam_sensor3 = "RFC03" metadata = {"location-lat": 27.986065, "location-long": 86.922623, "vehicle_id": "abg"} # Create calibration calibration_spec = create_sensor_calibration(f"Collection {datetime.now()}", [lidar_sensor1], [cam_sensor1, cam_sensor2, cam_sensor3]) created_calibration = client.calibration.create_calibration(calibration_spec) scene = LCSM.LidarsAndCamerasSequence( external_id=f"LCS-full-example-{uuid4()}", frames=[ LCSM.Frame( frame_id="1", relative_timestamp=0, point_clouds=[ PointCloud(filename="./examples/resources/point_cloud_RFL01.las", sensor_name=lidar_sensor1), ], images=[ Image(filename="./examples/resources/img_RFC01.jpg", sensor_name=cam_sensor1), Image(filename="./examples/resources/img_RFC02.jpg", sensor_name=cam_sensor2), ], metadata={"dut_status": "active"}, ego_vehicle_pose=EgoVehiclePose( position=Position(x=1.0, y=1.0, z=1.0), rotation=RotationQuaternion(w=0.01, x=1.01, y=1.01, z=1.01) ), ), LCSM.Frame( frame_id="2", relative_timestamp=500, point_clouds=[ PointCloud(filename="./examples/resources/point_cloud_RFL02.las", sensor_name=lidar_sensor1), ], images=[ Image(filename="./examples/resources/img_RFC11.jpg", sensor_name=cam_sensor1), Image(filename="./examples/resources/img_RFC12.jpg", sensor_name=cam_sensor2), ], ego_vehicle_pose=EgoVehiclePose( position=Position(x=2.0, y=2.0, z=2.0), rotation=RotationQuaternion(w=0.01, x=2.01, y=2.01, z=2.01) ), ), ], calibration_id=created_calibration.id, metadata=metadata, ) # Create scene return client.lidars_and_cameras_sequence.create(scene, dryrun=dryrun, **kwargs) if __name__ == "__main__": setup_logging(level="INFO") client = KognicIOClient() # Project - Available via `client.project.get_projects()` project = "" run(client, project=project) ``` :::hint{type="info"} ### Coordinate systems Note that both `position` and `rotation` for ego vehicle pose are with respect to the *local* coordinate system. ::: ## Shutter timings Shutter timings may be provided on `Image`s within `Frame`s. Shutter timings are a pair of nanosecond Unix timestamps values, both of which must be provided: shutter start and end time. They are specified for each image in each frame. ```python from __future__ import absolute_import import os.path from datetime import datetime from typing import Optional from uuid import uuid4 import kognic.io.model.scene.lidars_and_cameras_sequence as LCSM from examples.calibration.calibration import create_sensor_calibration from examples.imu_data.create_imu_data import create_dummy_imu_data from kognic.io.client import KognicIOClient from kognic.io.logger import setup_logging from kognic.io.model import CreateSceneResponse, Image, ImageMetadata, PointCloud def run(client: KognicIOClient, dryrun: bool = True, **kwargs) -> Optional[CreateSceneResponse]: print("Creating Lidar and Camera Sequence Scene...") lidar_sensor1 = "RFL01" lidar_sensor2 = "RFL02" cam_sensor1 = "RFC01" cam_sensor2 = "RFC02" metadata = {"location-lat": 27.986065, "location-long": 86.922623, "vehicleId": "abg"} examples_path = os.path.dirname(__file__) # Create calibration calibration_spec = create_sensor_calibration(f"Collection {datetime.now()}", [lidar_sensor1, lidar_sensor2], [cam_sensor1, cam_sensor2]) created_calibration = client.calibration.create_calibration(calibration_spec) # Generate IMU data ONE_MILLISECOND = 1000000 # one millisecond, expressed in nanos start_ts = 1648200140000000000 end_ts = start_ts + 10 * ONE_MILLISECOND imu_data = create_dummy_imu_data(start_timestamp=start_ts, end_timestamp=end_ts, samples_per_sec=1000) scene = LCSM.LidarsAndCamerasSequence( external_id=f"LCS-full-with-imu-and-shutter-example-{uuid4()}", frames=[ LCSM.Frame( frame_id="1", unix_timestamp=start_ts + ONE_MILLISECOND, relative_timestamp=0, point_clouds=[ PointCloud(filename=examples_path + "/resources/point_cloud_RFL01.csv", sensor_name=lidar_sensor1), PointCloud(filename=examples_path + "/resources/point_cloud_RFL02.csv", sensor_name=lidar_sensor2), ], images=[ Image( filename=examples_path + "/resources/img_RFC01.jpg", sensor_name=cam_sensor1, metadata=ImageMetadata( shutter_time_start_ns=start_ts + 0.5 * ONE_MILLISECOND, shutter_time_end_ns=start_ts + 1.5 * ONE_MILLISECOND ), ), Image( filename=examples_path + "/resources/img_RFC02.jpg", sensor_name=cam_sensor2, metadata=ImageMetadata( shutter_time_start_ns=start_ts + 0.5 * ONE_MILLISECOND, shutter_time_end_ns=start_ts + 1.5 * ONE_MILLISECOND ), ), ], ), LCSM.Frame( frame_id="2", unix_timestamp=start_ts + 5 * ONE_MILLISECOND, relative_timestamp=4, point_clouds=[ PointCloud(filename=examples_path + "/resources/point_cloud_RFL11.csv", sensor_name=lidar_sensor1), PointCloud(filename=examples_path + "/resources/point_cloud_RFL12.csv", sensor_name=lidar_sensor2), ], images=[ Image( filename=examples_path + "/resources/img_RFC11.jpg", sensor_name=cam_sensor1, metadata=ImageMetadata( shutter_time_start_ns=start_ts + 4.5 * ONE_MILLISECOND, shutter_time_end_ns=start_ts + 5.5 * ONE_MILLISECOND ), ), Image( filename=examples_path + "/resources/img_RFC12.jpg", sensor_name=cam_sensor2, metadata=ImageMetadata( shutter_time_start_ns=start_ts + 4.5 * ONE_MILLISECOND, shutter_time_end_ns=start_ts + 5.5 * ONE_MILLISECOND ), ), ], ), ], calibration_id=created_calibration.id, metadata=metadata, imu_data=imu_data, ) # Create scene return client.lidars_and_cameras_sequence.create(scene, dryrun=dryrun, **kwargs) if __name__ == "__main__": setup_logging(level="INFO") client = KognicIOClient() # Project - Available via `client.project.get_projects()` project = "" run(client, project=project) ``` ### Reverse rolling shutter :::hint{type="info"} Available since Kognic IO 2.17.0. ::: There is a flag `reversed_shutter` that can be set to `True` on `Images` when shutter timings are set, for cases where the camera's rolling shutter runs bottom-to-top. For example if the camera was mounted inverted; the pixels in the top row of the rotated image were captured later than pixels in the bottom row. It is important for our annotation tool to know the difference, in order to get the best quality of projections in the annotation tool. ```python ImageMetadata( shutter_time_start_ns=1775128684200000000, shutter_time_end_ns=1775128685000000000, reversed_shutter=True ) ```