Importing files from a volume

The Seven Bridges API allows you to import files from a volume in bulk rather than one by one. Using the bulk API feature, you can import up to 100 files per call.

Not optimized for rate limit

Importing individual files requires two API calls for each file: one to find the file by name, and another one to import it. We recommend using the bulk API call instead.

for f in files_to_import:
     
    imported_file = api.imports.submit_import(
        volume=volume,
        project=dest_project,
        location='christian_demo_files/' + f,
        overwrite=True
    )

Optimized for rate limit

Using the bulk API feature, you will first query all files that need to be imported and then use one API to import up to 100 files.

def bulk_import_files(file_names, volume, location, project, overwrite=True):
    """
    Imports list of files from volume in bulk.
    """

    chunk_size = 100  # Max legal bulk size for import is 100 items.
    final_responses = []

    def is_finished(response):
        return response in ["COMPLETED", "FAILED", "ABORTED"]

    def error_handling_after_completion(responses):
        errors = [s.resource.error.message for s in responses
                  if s.resource.state == "FAILED"]
        if errors:
            data = [
                s.resource.error
                if s.resource.state == "FAILED"
                else s.resource.result
                for s in responses
            ]
            raise Exception(
                'There were errors with bulk exporting.\n' +
                '\n'.join([str(d) for d in data])
            )

    def error_handling_after_submission(responses):
        errors = [s.error.message for s in responses if not s.valid]
        if errors:
            data = [s for s in responses if not s.valid]

            raise Exception(
                'There were errors with bulk submission.\n' +
                '\n'.join([
                    f'<Error: status={s.error.status}, code={s.error.code}>; '
                    f'{s.error.message}'
                    for s in data
                ])
            )

    # import files in batches of 100 each
    for i in range(0, len(file_names), chunk_size):

        # setup list of dictionary with import requests
        imports = [
            {
                'volume': volume,
                'location': location + '/' + fn,
                'project': project,
                'name': fn,
                'overwrite': overwrite
            }
            for fn in file_names[i:i + chunk_size]
        ]

        # initiate bulk import of batch and wait until finished
        responses = api.imports.bulk_submit(imports)

        # check for errors in bulk submission
        error_handling_after_submission(responses)

        # wait for bulk job to finish
        while not all(is_finished(s.resource.state)
                      for s in responses):
            time.sleep(10)
            responses = api.imports.bulk_get(
                [s.resource for s in responses]
            )

        # check if each job finished successfully
        error_handling_after_completion(responses)

        final_responses.extend(responses)

    return final_responses
 
 
responses = bulk_import_files(
    file_names=files_to_import,
    volume=volume,
    location="christian_demo_files",
    project=dest_project
)