Lidars and Cameras Sequence
6 min
Since kognic-io 2.5.0 we are promoting our newer scene model, which does not expose scene types (LidarsAndCamerasSequence) and simplifies creation of any scene from a single model.
A LidarsAndCamerasSequence consists of a sequence of camera images and lidar point clouds, where each frame consists on 1-20 camera images as well as 1-20 point clouds. For more documentation on what each field corresponds to in the LidarsAndCamerasSequence object please check the section related to Scene Overview.
from __future__ import absolute_import
from datetime import datetime
from typing import Optional
from uuid import uuid4
import kognic.io.model.scene.lidars_and_cameras_sequence as LCSM
from examples.calibration.calibration import create_sensor_calibration
from kognic.io.client import KognicIOClient
from kognic.io.logger import setup_logging
from kognic.io.model import CreateSceneResponse, Image, PointCloud
def run(client: KognicIOClient, dryrun: bool = True, **kwargs) -> Optional[CreateSceneResponse]:
print("Creating Lidar and Camera Sequence Scene...")
lidar_sensor1 = "lidar"
cam_sensor1 = "RFC01"
cam_sensor2 = "RFC02"
metadata = {"location-lat": 27.986065, "location-long": 86.922623, "vehicle_id": "abg"}
# Create calibration
# (Please refer to the API documentation about calibration for more details)
calibration_spec = create_sensor_calibration(
f"Collection {datetime.now()}",
[lidar_sensor1],
[cam_sensor1, cam_sensor2],
)
created_calibration = client.calibration.create_calibration(calibration_spec)
scene = LCSM.LidarsAndCamerasSequence(
external_id=f"LCS-example-{uuid4()}",
frames=[
LCSM.Frame(
frame_id="1",
relative_timestamp=0,
point_clouds=[
PointCloud(
filename="./examples/resources/point_cloud_RFL01.las",
sensor_name=lidar_sensor1,
),
],
images=[
Image(
filename="./examples/resources/img_RFC01.jpg",
sensor_name=cam_sensor1,
),
Image(
filename="./examples/resources/img_RFC02.jpg",
sensor_name=cam_sensor2,
),
],
metadata={"dut_status": "active"},
),
LCSM.Frame(
frame_id="2",
relative_timestamp=100,
point_clouds=[
PointCloud(
filename="./examples/resources/point_cloud_RFL02.las",
sensor_name=lidar_sensor1,
),
],
images=[
Image(
filename="./examples/resources/img_RFC11.jpg",
sensor_name=cam_sensor1,
),
Image(
filename="./examples/resources/img_RFC12.jpg",
sensor_name=cam_sensor2,
),
],
metadata={"dut_status": "active"},
),
],
calibration_id=created_calibration.id,
metadata=metadata,
)
# Create scene
return client.lidars_and_cameras_sequence.create(scene, dryrun=dryrun, **kwargs)
if __name__ == "__main__":
setup_logging(level="INFO")
client = KognicIOClient()
# Project - Available via `client.project.get_projects()`
project = "<project-identifier>"
run(client, project)Use dryrun to validate scene
Setting dryrun parameter to true in the method call, will validate the scene using the API but not create it.
Reuse calibration
Note that you can, and should, reuse the same calibration for multiple scenes if possible.
Providing Ego Vehicle Motion Information
Ego vehicle motion (i.e. the position and rotation of the ego vehicle) is optional information that can be provided when creating LidarsAndCamerasSeqs. This information can enable a massive reduction in the time it takes to annotate static objects. Ego vehicle motion information is provided by passing a EgoVehicleMotion object to each Framein the scene.
from __future__ import absolute_import
from datetime import datetime
from typing import Optional
from uuid import uuid4
import kognic.io.model.scene.lidars_and_cameras_sequence as LCSM
from examples.calibration.calibration import create_sensor_calibration
from kognic.io.client import KognicIOClient
from kognic.io.logger import setup_logging
from kognic.io.model import CreateSceneResponse, EgoVehiclePose, Image, PointCloud, Position, RotationQuaternion
def run(client: KognicIOClient, dryrun: bool = True, **kwargs) -> Optional[CreateSceneResponse]:
print("Creating Lidar and Camera Sequence Scene...")
lidar_sensor1 = "lidar"
cam_sensor1 = "RFC01"
cam_sensor2 = "RFC02"
cam_sensor3 = "RFC03"
metadata = {"location-lat": 27.986065, "location-long": 86.922623, "vehicle_id": "abg"}
# Create calibration
calibration_spec = create_sensor_calibration(f"Collection {datetime.now()}", [lidar_sensor1], [cam_sensor1, cam_sensor2, cam_sensor3])
created_calibration = client.calibration.create_calibration(calibration_spec)
scene = LCSM.LidarsAndCamerasSequence(
external_id=f"LCS-full-example-{uuid4()}",
frames=[
LCSM.Frame(
frame_id="1",
relative_timestamp=0,
point_clouds=[
PointCloud(filename="./examples/resources/point_cloud_RFL01.las", sensor_name=lidar_sensor1),
],
images=[
Image(filename="./examples/resources/img_RFC01.jpg", sensor_name=cam_sensor1),
Image(filename="./examples/resources/img_RFC02.jpg", sensor_name=cam_sensor2),
],
metadata={"dut_status": "active"},
ego_vehicle_pose=EgoVehiclePose(
position=Position(x=1.0, y=1.0, z=1.0), rotation=RotationQuaternion(w=0.01, x=1.01, y=1.01, z=1.01)
),
),
LCSM.Frame(
frame_id="2",
relative_timestamp=500,
point_clouds=[
PointCloud(filename="./examples/resources/point_cloud_RFL02.las", sensor_name=lidar_sensor1),
],
images=[
Image(filename="./examples/resources/img_RFC11.jpg", sensor_name=cam_sensor1),
Image(filename="./examples/resources/img_RFC12.jpg", sensor_name=cam_sensor2),
],
ego_vehicle_pose=EgoVehiclePose(
position=Position(x=2.0, y=2.0, z=2.0), rotation=RotationQuaternion(w=0.01, x=2.01, y=2.01, z=2.01)
),
),
],
calibration_id=created_calibration.id,
metadata=metadata,
)
# Create scene
return client.lidars_and_cameras_sequence.create(scene, dryrun=dryrun, **kwargs)
if __name__ == "__main__":
setup_logging(level="INFO")
client = KognicIOClient()
# Project - Available via `client.project.get_projects()`
project = "<project-identifier>"
run(client, project=project)Coordinate systems
Note that both position and rotation for ego vehicle pose are with respect to the local coordinate system.
Shutter timings
Shutter timings may be provided on Images within Frames.
Shutter timings are a pair of nanosecond Unix timestamps values, both of which must be provided: shutter start and end time. They are specified for each image in each frame.
from __future__ import absolute_import
import os.path
from datetime import datetime
from typing import Optional
from uuid import uuid4
import kognic.io.model.scene.lidars_and_cameras_sequence as LCSM
from examples.calibration.calibration import create_sensor_calibration
from examples.imu_data.create_imu_data import create_dummy_imu_data
from kognic.io.client import KognicIOClient
from kognic.io.logger import setup_logging
from kognic.io.model import CreateSceneResponse, Image, ImageMetadata, PointCloud
def run(client: KognicIOClient, dryrun: bool = True, **kwargs) -> Optional[CreateSceneResponse]:
print("Creating Lidar and Camera Sequence Scene...")
lidar_sensor1 = "RFL01"
lidar_sensor2 = "RFL02"
cam_sensor1 = "RFC01"
cam_sensor2 = "RFC02"
metadata = {"location-lat": 27.986065, "location-long": 86.922623, "vehicleId": "abg"}
examples_path = os.path.dirname(__file__)
# Create calibration
calibration_spec = create_sensor_calibration(f"Collection {datetime.now()}", [lidar_sensor1, lidar_sensor2], [cam_sensor1, cam_sensor2])
created_calibration = client.calibration.create_calibration(calibration_spec)
# Generate IMU data
ONE_MILLISECOND = 1000000 # one millisecond, expressed in nanos
start_ts = 1648200140000000000
end_ts = start_ts + 10 * ONE_MILLISECOND
imu_data = create_dummy_imu_data(start_timestamp=start_ts, end_timestamp=end_ts, samples_per_sec=1000)
scene = LCSM.LidarsAndCamerasSequence(
external_id=f"LCS-full-with-imu-and-shutter-example-{uuid4()}",
frames=[
LCSM.Frame(
frame_id="1",
unix_timestamp=start_ts + ONE_MILLISECOND,
relative_timestamp=0,
point_clouds=[
PointCloud(filename=examples_path + "/resources/point_cloud_RFL01.csv", sensor_name=lidar_sensor1),
PointCloud(filename=examples_path + "/resources/point_cloud_RFL02.csv", sensor_name=lidar_sensor2),
],
images=[
Image(
filename=examples_path + "/resources/img_RFC01.jpg",
sensor_name=cam_sensor1,
metadata=ImageMetadata(
shutter_time_start_ns=start_ts + 0.5 * ONE_MILLISECOND, shutter_time_end_ns=start_ts + 1.5 * ONE_MILLISECOND
),
),
Image(
filename=examples_path + "/resources/img_RFC02.jpg",
sensor_name=cam_sensor2,
metadata=ImageMetadata(
shutter_time_start_ns=start_ts + 0.5 * ONE_MILLISECOND, shutter_time_end_ns=start_ts + 1.5 * ONE_MILLISECOND
),
),
],
),
LCSM.Frame(
frame_id="2",
unix_timestamp=start_ts + 5 * ONE_MILLISECOND,
relative_timestamp=4,
point_clouds=[
PointCloud(filename=examples_path + "/resources/point_cloud_RFL11.csv", sensor_name=lidar_sensor1),
PointCloud(filename=examples_path + "/resources/point_cloud_RFL12.csv", sensor_name=lidar_sensor2),
],
images=[
Image(
filename=examples_path + "/resources/img_RFC11.jpg",
sensor_name=cam_sensor1,
metadata=ImageMetadata(
shutter_time_start_ns=start_ts + 4.5 * ONE_MILLISECOND, shutter_time_end_ns=start_ts + 5.5 * ONE_MILLISECOND
),
),
Image(
filename=examples_path + "/resources/img_RFC12.jpg",
sensor_name=cam_sensor2,
metadata=ImageMetadata(
shutter_time_start_ns=start_ts + 4.5 * ONE_MILLISECOND, shutter_time_end_ns=start_ts + 5.5 * ONE_MILLISECOND
),
),
],
),
],
calibration_id=created_calibration.id,
metadata=metadata,
imu_data=imu_data,
)
# Create scene
return client.lidars_and_cameras_sequence.create(scene, dryrun=dryrun, **kwargs)
if __name__ == "__main__":
setup_logging(level="INFO")
client = KognicIOClient()
# Project - Available via `client.project.get_projects()`
project = "<project-id>"
run(client, project=project)Reverse rolling shutter
Available since Kognic IO 2.17.0.
There is a flag reversed_shutter that can be set to True on Images when shutter timings are set, for cases where the camera's rolling shutter runs bottom-to-top. For example if the camera was mounted inverted; the pixels in the top row of the rotated image were captured later than pixels in the bottom row. It is important for our annotation tool to know the difference, in order to get the best quality of projections in the annotation tool.
ImageMetadata(
shutter_time_start_ns=1775128684200000000,
shutter_time_end_ns=1775128685000000000,
reversed_shutter=True
)