How to: Upload (Real-time)

import math
from datetime import datetime, timezone
from typing import Callable

# The minimum size, in bytes, for a single, non-final part upload.
MINIMUM_PART_SIZE = 5_242_880
# The maximum filesize in
MAXIMUM_PART_COUNT = 10_000
# The maximum size, in bytes, for an AWS upload.
MAXIMUM_FILE_SIZE = 5_497_558_138_880

# The data rate at which every part is an equal size, and could not
# be any uniformly larger without violating the maximum total file
# size if 10_000 parts were to be uploaded. it works out to
# ~549.8 MB per payload. By enforcing this we actually never need
# to check if a part exceeds the maximum allowed part size, as our
# parts will never exceed ~549.8 MB.
MAXIMUM_DATA_RATE = MAXIMUM_FILE_SIZE // MAXIMUM_PART_COUNT

def create_part_size_calculator(format_bytes_per_second: int) -> Callable[[int], int]:
    """
    Returns a function that takes in a `part_number` and returns a
    `part_size` based on `data_rate`.
    """
    ...

def upload_render(data_stream: DataStream, channel: int = 0) -> None:
    """
    Uploads an asset for data_stream, which is a custom IO class that pulls remaining
    upload data from an internal buffer or file, depending on how well the upload is
    keeping pace with the render.

    Uploads to `channel`
    """

    asset = c2c.asset_create(
        extension=data_stream.extension, 
        filetype=data_stream.mimetpye, 
        channel=channel,
        offset=datetime.now(timezone.utc) - data_stream.created_at()
    )

    calculate_part_size = create_part_size_calculator(data_stream.data_rate())
    next_part_number = 0

    while True:
        next_payload_size = calculate_part_size(next_part_number)

        # Waits until one or more chunks worth of data is ready for upload. Cache 
        # whether our data stream has completed writing the file, and the current 
        # number of bytes we have remaining to upload at this time.
        available_bytes, stream_complete = data_stream.wait_for_available_data(
            minimum_bytes=next_payload_size
        )

        # Build the list of parts to request based on our available data.
        parts = []
        while available_bytes > 0:
            payload_size = calculate_part_size(next_part_number)

            if available_bytes < payload_size and not stream_complete:
                break

            payload_size = min(payload_size, available_bytes)

            parts.append(
                c2c.RealtimeUploadPart(
                    part_number=next_part_number,
                    part_size=payload_size,
                    is_final=False
                )
            )

            available_bytes -= payload_size
            next_part_number += 1

        # If our stream is done writing, mark the last part as final.
        if stream_complete:
            parts[-1].is_final = True

        # Create the part URLs using the C2C endpoint.
        response = c2c.create_realtime_parts(
            asset_id=asset.id,
            asset_name=None if not stream_complete else data_stream.filename,
            asset_filesize=None if not stream_complete else data_stream.size(),
            parts=parts
        )

        # Upload each part to its URL.
        for part, part_url in zip(parts, response.upload_urls):
            part_data = data_stream.read(bytes=part.size)
            c2c.upload_chunk(part_data, part_url, data_stream.mimetype)

        if stream_complete:
            break