Importing files from a volume

for f in files_to_import:
     
    imported_file = api.imports.submit_import(
        volume=volume,
        project=dest_project,
        location='christian_demo_files/' + f,
        overwrite=True
    )

def bulk_import_files(file_names, volume, location, project, overwrite=True):
    """
    Imports list of files from volume in bulk.
    """

    chunk_size = 100  # Max legal bulk size for import is 100 items.
    final_responses = []

    def is_finished(response):
        return response in ["COMPLETED", "FAILED", "ABORTED"]

    def error_handling_after_completion(responses):
        errors = [s.resource.error.message for s in responses
                  if s.resource.state == "FAILED"]
        if errors:
            data = [
                s.resource.error
                if s.resource.state == "FAILED"
                else s.resource.result
                for s in responses
            ]
            raise Exception(
                'There were errors with bulk exporting.\n' +
                '\n'.join([str(d) for d in data])
            )

    def error_handling_after_submission(responses):
        errors = [s.error.message for s in responses if not s.valid]
        if errors:
            data = [s for s in responses if not s.valid]

            raise Exception(
                'There were errors with bulk submission.\n' +
                '\n'.join([
                    f'<Error: status={s.error.status}, code={s.error.code}>; '
                    f'{s.error.message}'
                    for s in data
                ])
            )

    # import files in batches of 100 each
    for i in range(0, len(file_names), chunk_size):

        # setup list of dictionary with import requests
        imports = [
            {
                'volume': volume,
                'location': location + '/' + fn,
                'project': project,
                'name': fn,
                'overwrite': overwrite
            }
            for fn in file_names[i:i + chunk_size]
        ]

        # initiate bulk import of batch and wait until finished
        responses = api.imports.bulk_submit(imports)

        # check for errors in bulk submission
        error_handling_after_submission(responses)

        # wait for bulk job to finish
        while not all(is_finished(s.resource.state)
                      for s in responses):
            time.sleep(10)
            responses = api.imports.bulk_get(
                [s.resource for s in responses]
            )

        # check if each job finished successfully
        error_handling_after_completion(responses)

        final_responses.extend(responses)

    return final_responses
 
 
responses = bulk_import_files(
    file_names=files_to_import,
    volume=volume,
    location="christian_demo_files",
    project=dest_project
)