#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Created on January 14th, 2015: initial commit -- XnatNDAR creation
Edited on April 14th, 2015: second commit -- add processed data

@author: Benjamin Yvernault, Electrical Engineering, Vanderbilt University

Script to generate the folder for a NDAR submission using one of these options:
 1) the template image03 for raw data
 2) the template fmriresults01 for processed data.
You need to specify with the csv options which one you want to upload.

You can find the two templates on NDAR website at:
https://ndar.nih.gov/ndar_data_dictionary.html
The Inputs changed if you choose 1) or 2).
You can specify 1 or 2 by setting the options for the assessor csv or the scan
csv.

Other Inputs:
    directory: temp directory where the data will be downloaded
    project: id for the project on XNAT
    subject info: CSV file with the following header:

    subject_id_xnat,GUID,study_subject_id,interview_date,interview_age,gender

    This csv allows the script to know which subject to download from XNAT and
    what are the information like gender (not accessible on XNAT).

Inputs if template 1, raw data:
    scan info: CSV with the following header:

    xnat_scantype,xnat_series_description,assessor_type_qc,ndar_scantype,\
image_num_dimensions,slice_acquisition,pulse_sequence,citation,experiment_id,\
image_file,image_thumbnail_file,data_file2,data_file2_definition

    This csv allows the script to know some generic information for the
    template that is not on XNAT or in the DICOM.

Inputs if template 2, processed data:
    assessor info: CSV with the following header:

    object_type,proc_types,inputs,metric_files_xnat,derived_files_xnat,\
pipeline,pipeline_script,pipeline_tools,pipeline_version

    This csv allows the script to know some generic information on processed
    data for the template that is not on XNAT or in the DICOM.
    proc_types = assessor type or scan type depending on the object_type
    ("assessor" or "scan")

Example of values below for T1,fMRI,dti:

xnat_scantype,xnat_series_description,assessor_type_qc,ndar_scantype,image_num_dimensions,slice_acquisition,pulse_sequence,citation,experiment_id,image_file,image_thumbnail_file,data_file2,data_file2_definition
T1,,VBMQA,MR structural,3,,MPRAGE(GE),\
DOI: 10.2174/1573405054038726,,NIFTI,SNAPSHOTS,,
fMRI,,fMRIQA,fMRI,4,4,BOLD T2* EPI (GE),\
DOI: 10.1002/jmri.20583,20,NIFTI,SNAPSHOTS,,
dti,,dtiQA_v2,MR diffusion,4,4,DTI Stejskal-Tanner (SE),\
DOI: 10.1371/journal.pone.0061737,,NIFTI,SNAPSHOTS,"bval,bvec",\
bval & bvec (.zip)

Example of values below for assessor (FS,fMRIQA,Multi_Atlas and skull strip
T1):

object_type,proc_types,inputs,metric_files_xnat,derived_files_xnat,pipeline,pipeline_script,pipeline_tools,pipeline_version,experiment_id,scan_type
assessor,FreeSurfer,MR Structural (T1/MPRAGE) NIFTI,DATA, ,\
FreeSurfer,Spiders_FreeSurferRecon_v4.0.2.py,FreeSurfer_v5.1,4.0.2,,MR \
structural (T1)
assessor,fMRIQA,fMRI NIFTI,STATS, ,fMRIQA,Spiders_fMRIQA.py,Matlab_r2013a,\
1.0.0,,201,fMRI
assessor,Multi_Atlas,MR Structural (T1/MPRAGE) NIFTI,VOL_TXT, ,Multi Atlas, \
Spiders_Multi_Atlas.py,"Matlab_r2013a,JIST,MIPAV",1.0.0,,MR structural (T1)
scan,T1,MR Structural (T1/MPRAGE) NIFTI and Multi Atlas \
Segmentation,,NIFTI_noskull,Module_T1_skull_strip, \
Module_T1_skull_strip.py,Matlab_2013,1.0.0,,MR structural (T1)

"""

from __future__ import print_function
from __future__ import division

from builtins import next
from builtins import str
from builtins import zip
from builtins import range
from past.builtins import basestring

import os
import csv
import sys
import glob
import dicom
import shutil
from dicom.tag import Tag
from dax import XnatUtils


try:
    basestring
except NameError:
    basestring = str

DEFAULT_SCAN_HEADER = ['xnat_scantype', 'xnat_series_description',
                       'assessor_type_qc', 'ndar_scantype',
                       'image_num_dimensions', 'slice_acquisition',
                       'pulse_sequence', 'citation', 'experiment_id',
                       'image_file', 'image_thumbnail_file', 'data_file2',
                       'data_file2_definition']
DEFAULT_ASSESSOR_HEADER = ['object_type', 'proc_types', 'inputs',
                           'metric_files_xnat', 'derived_files_xnat',
                           'pipeline', 'pipeline_script', 'pipeline_tools',
                           'pipeline_version', 'experiment_id', 'scan_type']
DEFAULT_SUBJECT_HEADER = ['xnat_subject_id', 'GUID', 'study_subject_id',
                          'interview_date', 'interview_age', 'gender']
SCAN_ORDERED_KEYS = ['subjectkey', 'src_subject_id', 'interview_date',
                     'interview_age', 'gender', 'comments_misc', 'image_file',
                     'image_thumbnail_file', 'image_description',
                     'experiment_id', 'scan_type', 'scan_object',
                     'image_file_format', 'data_file2',
                     'data_file2_type', 'image_modality',
                     'scanner_manufacturer_pd', 'scanner_type_pd',
                     'scanner_software_versions_pd',
                     'magnetic_field_strength', 'mri_repetition_time_pd',
                     'mri_echo_time_pd',
                     'flip_angle', 'acquisition_matrix',
                     'mri_field_of_view_pd', 'patient_position',
                     'photomet_interpret', 'receive_coil', 'transmit_coil',
                     'transformation_performed', 'transformation_type',
                     'image_history', 'image_num_dimensions',
                     'image_extent1', 'image_extent2',
                     'image_extent3', 'image_extent4', 'extent4_type',
                     'image_extent5', 'extent5_type',
                     'image_unit1', 'image_unit2', 'image_unit3',
                     'image_unit4', 'image_unit5',
                     'image_resolution1', 'image_resolution2',
                     'image_resolution3', 'image_resolution4',
                     'image_resolution5', 'image_slice_thickness',
                     'image_orientation',
                     'qc_outcome', 'qc_description', 'qc_fail_quest_reason',
                     'decay_correction', 'frame_end_times', 'frame_end_unit',
                     'frame_start_times', 'frame_start_unit', 'pet_isotope',
                     'pet_tracer', 'time_diff_inject_to_image',
                     'time_diff_units', 'pulse_seq', 'slice_acquisition',
                     'software_preproc']
SCAN_HEADER = ['image', '03']
NDAR_SCAN_CSV_DICT = {
    'subjectkey': 'options',
    'src_subject_id': 'options',
    'interview_date': 'options',
    'interview_age': 'options',
    'gender': 'options',
    'comments_misc': '',
    'image_file': 'XNAT',
    'image_thumbnail_file': 'XNAT',
    'image_description': 'XNAT',
    'experiment_id': 'options',
    'scan_type': 'options',
    'scan_object': 'Live',
    'image_file_format': 'NIFTI',
    'data_file2': 'options',
    'data_file2_type': 'options',
    'image_modality': 'MRI',
    'scanner_manufacturer_pd': ('0008', '0070'),
    'scanner_type_pd': ('0008', '1090'),
    'scanner_software_versions_pd': ('0018', '1020'),
    'magnetic_field_strength': ('0018', '0087'),
    'mri_repetition_time_pd': ('0018', '0080'),
    'mri_echo_time_pd': ('0018', '0081'),
    'flip_angle': ('2001', '1023'),
    'acquisition_matrix': 'ds',
    'mri_field_of_view_pd': '',
    'patient_position': ('0018', '5100'),
    'photomet_interpret': ('0028', '0004'),
    'receive_coil': 'ds',
    'transmit_coil': 'ds',
    'transformation_performed': 'No',
    'transformation_type': '',
    'image_history': '',
    'image_num_dimensions': 'options',
    'image_extent1': ('0028', '0010'),
    'image_extent2': ('0028', '0011'),
    'image_extent3': 'ds',
    'image_extent4': 'ds',
    'extent4_type': 'ds',
    'image_extent5': '',
    'extent5_type': '',
    'image_unit1': 'frame number',
    'image_unit2': 'frame number',
    'image_unit3': 'frame number',
    'image_unit4': 'frame number',
    'image_unit5': '',
    'image_resolution1': 'ds',
    'image_resolution2': 'ds',
    'image_resolution3': 'ds',
    'image_resolution4': 'ds',
    'image_resolution5': '0',
    'image_slice_thickness': ('0018', '0088'),
    'image_orientation': 'ds',
    'qc_outcome': 'XNAT',
    'qc_description': 'options',
    'qc_fail_quest_reason': '',
    'decay_correction': '',
    'frame_end_times': '',
    'frame_end_unit': '',
    'frame_start_times': '',
    'frame_start_unit': '',
    'pet_isotope': '',
    'pet_tracer': '',
    'time_diff_inject_to_image': '',
    'time_diff_units': '',
    'pulse_seq': 'options',
    'slice_acquisition': 'options',
    'software_preproc': ''}
ASSESSOR_ORDERED_KEYS = [
    'subjectkey', 'src_subject_id', 'origin_dataset_id', 'interview_date',
    'interview_age', 'gender', 'experiment_id', 'inputs', 'img03_id',
    'file_source', 'job_name', 'proc_types', 'metric_files', 'pipeline',
    'pipeline_script', 'pipeline_tools', 'pipeline_type', 'pipeline_version',
    'qc_fail_quest_reason', 'qc_outcome', 'derived_files', 'scan_type']
ASSESSOR_HEADER = ['fmriresults', '01']
NDAR_ASSESSOR_CSV_DICT = {
    'subjectkey': 'options',
    'src_subject_id': 'options',
    'origin_dataset_id': '',
    'interview_date': 'options',
    'interview_age': 'options',
    'gender': 'options',
    'experiment_id': 'options',
    'inputs': 'options',
    'img03_id': '',
    'file_source': 'NDAR',
    'job_name': 'options',
    'proc_types': 'options',
    'metric_files': 'XNAT',
    'pipeline': 'options',
    'pipeline_script': 'options',
    'pipeline_tools': 'options',
    'pipeline_type': 'python version 2.7.X',
    'pipeline_version': 'options',
    'qc_fail_quest_reason': 'XNAT',
    'qc_outcome': 'XNAT',
    'derived_files': 'XNAT',
    'scan_type': 'options'}


def get_qc(assessor_list, assessor_label):
    """ Extract qc information for the scan from assessor """
    assessor = [a for a in assessor_list if a['label'] == assessor_label]
    return get_qc_assessor(assessor[0])


def get_qc_assessor(assessor):
    """ Extract qc information from assessor """
    qc_status = 'questionable'
    qc_reason = 'Not yet control by user'
    if assessor['qcstatus'] in ['Needs QA', 'JOB_PENDING']:
        pass
    elif 'fail' in assessor['qcstatus'].lower() or \
         'bad' in assessor['qcstatus'].lower():
        qc_status = 'fail'
        qc_reason = '-'
    else:
        qc_status = 'pass'
        qc_reason = 'n/a'
    return qc_status, qc_reason


def get_qc_proc_scan(scan):
    """ Extract qc information from scan """
    qc_status = 'questionable'
    qc_reason = 'Not yet control by user'
    if scan['quality'] == 'usable':
        qc_status = 'pass'
        qc_reason = 'n/a'
    elif scan['quality'] == 'unusable':
        qc_status = 'fail'
        qc_reason = 'scan not usable'
    else:
        pass

    return qc_status, qc_reason


def filter_list(values, label, obj_list):
    """ filter a list of dictionary to keep only the one with the label in
    values
    """
    if values:
        obj_list = [x for x in obj_list if x[label] in values]
    return obj_list


def get_slices_volumes(ds):
    """ Get number of slices and volumes from dicom """
    t = list()
    nFrames = ds[0x0028, 0x0008].value
    for i in range(0, nFrames):
        t.append(
            ds[0x5200, 0x9230][i][0x2005, 0x140f][0][0x2001, 0x100a].value)
    nSlices = max(t)
    nVols = nFrames / nSlices
    return nSlices, nVols


def getAxis(x, y, z):
    """ Get axis from figure """
    thresh = 0.8
    axis = ''
    ax = abs(x)
    ay = abs(y)
    az = abs(z)

    ox = 'R' if x < 0 else 'L'
    oy = 'A' if y < 0 else 'P'
    oz = 'F' if z < 0 else 'H'

    if ax > thresh and ax > ay and ax > az:
        axis = ox
    elif ay > thresh and ay > ax and ay > az:
        axis = oy
    elif az > thresh and az > ax and az > ay:
        axis = oz
    return axis


def getLabel(ds):
    """ Extract from dicom the orientation (Axial/Coronal/Sagittal)"""
    orientation = ds[0x5200, 0x9230][0][0x0020, 0x9116][0][0x0020, 0x0037].value
    a_row = getAxis(orientation[0], orientation[1], orientation[2])
    a_col = getAxis(orientation[3], orientation[4], orientation[5])
    label = ''
    if (a_row == 'R' or a_row == 'L') and (a_col == 'A' or a_col == 'P'):
        label = 'Axial'
    elif (a_col == 'R' or a_col == 'L') and (a_row == 'A' or a_row == 'P'):
        label = 'Axial'
    elif (a_row == 'R' or a_row == 'L') and (a_col == 'H' or a_col == 'F'):
        label = 'Coronal'
    elif (a_col == 'R' or a_col == 'L') and (a_row == 'H' or a_row == 'F'):
        label = 'Coronal'
    elif (a_row == 'A' or a_row == 'P') and (a_col == 'H' or a_col == 'F'):
        label = 'Sagittal'
    elif (a_col == 'A' or a_col == 'P') and (a_row == 'H' or a_row == 'F'):
        label = 'Sagittal'
    return label


def get_scan_row(directory, subject_record, scan_record):
    """ Convert records into a row ordered to print to csv for scan"""
    row = list()
    if 'DICOM' not in list(scan_record.keys()):
        print('  ---> warning: no dicom found')
    else:
        if len(scan_record['DICOM']) > 0 and scan_record['DICOM'][0] == '/':
            dcmpath = os.path.join(directory, scan_record['DICOM'][1:])
        else:
            dcmpath = os.path.join(directory, scan_record['DICOM'])
        if not os.path.isfile(dcmpath):
            print('  ---> warning: dicom missing')
        else:
            ds = dicom.read_file(dcmpath)
            # read the keys in order
            for header in SCAN_ORDERED_KEYS:
                # DICOM header tuple
                if isinstance(NDAR_SCAN_CSV_DICT[header], tuple):
                    t2 = Tag(NDAR_SCAN_CSV_DICT[header][0],
                             NDAR_SCAN_CSV_DICT[header][1])
                    try:
                        val = ds[t2].value
                    except Exception:
                        val = ' '
                # Specific value
                elif isinstance(NDAR_SCAN_CSV_DICT[header], basestring):
                    val = get_value(header, subject_record, scan_record, ds)
                else:
                    val = ''
                # Add the value to the row
                row.append(str(val))

    return row


def get_assessor_row(directory, subject_record, proc_record):
    """ Convert records into a row ordered to print to csv for assessor"""
    row = list()
    _skeys = list(subject_record.keys())
    _pkeys = list(proc_record.keys())
    for header in ASSESSOR_ORDERED_KEYS:
        if header in _skeys:
            val = subject_record[header]
        elif header == 'job_name':
            val = proc_record['label']
        elif header == 'subjectkey' and 'GUID' in _skeys:
            val = subject_record['GUID']
        elif header == 'src_subject_id' and 'study_subject_id' in _skeys:
            val = subject_record['study_subject_id']
        elif header == 'interview_date' and 'interview_date' in _skeys:
            val = subject_record['interview_date']
        elif header == 'interview_age' and 'interview_age' in _skeys:
            val = subject_record['interview_age']
        elif header == 'gender' and 'gender' in _skeys:
            val = subject_record['gender']
        elif header == 'derived_files' and 'derived_files_xnat' in _pkeys:
            val = proc_record['derived_files_xnat']
        elif header == 'metric_files' and 'metric_files_xnat' in _pkeys:
            val = proc_record['metric_files_xnat']
        elif header in _pkeys:
            val = proc_record[header]
        else:
            val = NDAR_ASSESSOR_CSV_DICT[header]
        row.append(str(val))
    return row


def get_value(header, subject_record, scan_record, ds):
    """
    Get the value for a specific header from dicom / scan record or
    subject record
    """
    val = ''
    _skeys = list(subject_record.keys())
    _sckeys = list(scan_record.keys())
    if header == 'acquisition_matrix':
        val = ds[0x5200, 0x9230][0][0x2005, 0x140f][0][0x0018, 0x1310].value
    elif header == 'image_resolution1':
        val = ds[0x5200, 0x9230][0][0x2005, 0x140f][0][0x0028, 0x0030].value[0]
    elif header == 'image_resolution2':
        val = ds[0x5200, 0x9230][0][0x2005, 0x140f][0][0x0028, 0x0030].value[1]
    elif header == 'image_resolution3':
        val = ds[0x5200, 0x9230][0][0x2005, 0x140f][0][0x0018, 0x0050].value
    elif header == 'image_resolution4':
        val = ds[0x5200, 0x9229][0][0x0018, 0x9112][0][0x0018, 0x0080].value
    elif header == 'image_orientation':
        val = getLabel(ds)
    elif header == 'image_extent3':
        nSlices, nVols = get_slices_volumes(ds)
        val = nSlices
    elif header == 'image_extent4':
        nSlices, nVols = get_slices_volumes(ds)
        val = nVols
    elif header == 'receive_coil':
        val = ds[0x5200, 0x9229][0][0x0018, 0x9042][0][0x0018, 0x1250].value
    elif header == 'transmit_coil':
        val = ds[0x5200, 0x9229][0][0x0018, 0x9049][0][0x0018, 0x1251].value
    elif header == 'subjectkey' and 'GUID' in _skeys:
        val = subject_record['GUID']
    elif header == 'src_subject_id' and 'study_subject_id' in _skeys:
        val = subject_record['study_subject_id']
    elif header == 'interview_date' and 'interview_date' in _skeys:
        val = subject_record['interview_date']
    elif header == 'interview_age' and 'interview_age' in _skeys:
        val = subject_record['interview_age']
    elif header == 'gender' and 'gender' in _skeys:
        val = subject_record['gender']
    elif header == 'data_file2_type' and 'data_file2_definition' in _sckeys:
        val = scan_record['data_file2_definition']
    elif header == 'pulse_seq' and 'pulse_sequence' in _sckeys:
        val = scan_record['pulse_sequence']
    elif header == 'qc_description' and 'citation' in _sckeys:
        val = scan_record['citation']
    elif header == 'scan_type' and 'ndar_scantype' in _sckeys:
        val = scan_record['ndar_scantype']
    elif header == 'extent4_type':
        if 'ndar_scantype' in _sckeys and \
           scan_record['ndar_scantype'] == 'MR structural (T1)':
            val = ''
        else:
            val = 'Number of Volumes'
    elif header in _sckeys:
        val = scan_record[header]
    else:
        val = NDAR_SCAN_CSV_DICT[header]
    return val


def read_csv(csvpath, default_header):
    """ Read the csv to extract previous row compute """
    csv_dict = dict()
    with open(csvpath, 'rb') as csvfile:
        spamreader = csv.reader(csvfile, delimiter=',')
        keys = next(spamreader)
        # Check if the keys are the right header, if not, assume there was no
        # header:
        if set(keys) == set(default_header):
            pass
        else:
            csv_dict[keys[0]] = dict(list(zip(default_header, keys)))
            keys = default_header

        for row in spamreader:
            if default_header[0] == 'xnat_scantype':
                k = row[0] + '-x-' + row[1]
            elif default_header[0] == 'object_type':
                k = row[1]
            else:
                k = row[0]
            csv_dict[k] = dict(list(zip(keys, row)))

    return csv_dict


def write_csv(csv_fpath, csv_head, csv_headers, records, get_row,
              continu=False):
    """ Write the csv for scan or processed data from Assessor:
        csv_fpath: csv file path
        csv_head: first line of the csv (e.g: image,03)
        records: dictionary of information extracted from XNAT/DICOM/INPUTS ...
        get_row: function to get the row from the records
    """
    # Read Previous csv:
    previous_row = list()
    previous_GUID = list()

    # Continu options
    if continu and os.path.exists(outputcsv):
        with open(outputcsv, 'rb') as csvfile:
            spamreader = csv.reader(csvfile, delimiter=',')
            next(spamreader)  # remove the image,03
            next(spamreader)  # remove the header line
            for row in spamreader:
                previous_GUID.append(row[0])
                previous_row.append(row)

    with open(csv_fpath, 'wb') as csvfile:
        spamwriter = csv.writer(csvfile, delimiter=',')
        spamwriter.writerow(csv_head)
        spamwriter.writerow(csv_headers)

        if previous_row:
            for row in previous_row:
                spamwriter.writerow(row)

        print('INFO: Writting rows for the csv from DICOM header ...')
        for rec in records:
            guid = subject_records[rec['subject_label']]['GUID']
            if guid not in previous_GUID:
                mess = """ Subject: {subject} -- Session: {session} -- Scan/\
Assessor: {label}"""
                label = rec['ID'] if options.scan_inf else rec['label']
                print(mess.format(
                    subject=rec['subject_label'],
                    session=rec['session_label'],
                    label=label))
                row = get_row(
                    directory, subject_records[rec['subject_label']], rec)
                if row:
                    spamwriter.writerow(row)


def get_scan_xnat(options, xnat, directory, subjects):
    """ Extract information from XNAT for the csv for scan """
    scan_records = list()
    scan_info = read_csv(options.scaninfo, DEFAULT_SCAN_HEADER)

    # list of scans and assessors for the full project
    scan_list = XnatUtils.list_project_scans(xnat, options.project)
    assessor_list = XnatUtils.list_project_assessors(xnat, options.project)

    # filter to keep only the scan we need and keep the assessor that we need
    # for the qc Scans
    scantypes = [v['xnat_scantype'] for _, v in list(scan_info.items())]
    scan_list = filter_list(scantypes, 'type', scan_list)
    scan_list = filter_list(['usable'], 'quality', scan_list)
    scan_list = filter_list(subjects, 'subject_label', scan_list)
    # Assessors
    assessor_list = filter_list(subjects, 'subject_label', assessor_list)

    print('INFO: Downloading scans resources from XNAT')
    # For each scan, download the data and write the scan_records for the csv
    for scan in sorted(scan_list, key=lambda k: k['subject_label']):
        for type_SD in list(scan_info.keys()):
            scantype = type_SD.split('-x-')[0]
            part_of_SD = type_SD.split('-x-')[1]
            if scan['type'].lower() == scantype.lower() and \
               part_of_SD.lower() in scan['series_description'].lower():
                mess = """ subject/session/scan: {subj}/{sess}/{scan} found"""
                print(mess.format(subj=scan['subject_label'],
                                  sess=scan['session_label'],
                                  scan=scan['ID']))
                scan_dict = scan.copy()  # Copy the dict
                # for each header in the scan_info specific to this type of
                # scan :
                for header, value in list(scan_info[type_SD].items()):
                    if header in ['image_file', 'image_thumbnail_file',
                                  'data_file2'] and value:
                        scan_dict[header] = download_scan_file(
                            xnat, directory, scan, value.split(','), header)
                    elif header == 'assessor_type_qc':
                        # Add the qc outcome
                        assessor_label = '-x-'.join([scan['project_id'],
                                                     scan['subject_label'],
                                                     scan['session_label'],
                                                     scan['ID'],
                                                     value])
                        qc_status, qc_reason = get_qc(
                            assessor_list, assessor_label)
                        scan_dict['qc_outcome'] = qc_status
                        scan_dict['qc_fail_quest_reason'] = qc_reason
                    else:
                        scan_dict[header] = value
                # Download the DICOM
                scan_dict['DICOM'] = download_scan_file(
                    xnat, directory, scan, ['DICOM'], 'DICOM')
                scan_records.append(scan_dict)

    return scan_records


def extract_info(proc_info):
    """ Extracting two dictionaries from the dictionary proc_info """
    scan_info = dict()
    assessor_info = dict()
    for key, info in list(proc_info.items()):
        if info['object_type'] == 'scan':
            scan_info[key] = info
        elif info['object_type'] == 'assessor':
            assessor_info[key] = info
    return scan_info, assessor_info


def get_processed_data_xnat(options, xnat, directory, subjects):
    """ Extract information from XNAT for the csv for processed data
    (assessor or scan)
    """
    proc_info = read_csv(options.assessorinfo, DEFAULT_ASSESSOR_HEADER)
    records1 = list()
    records2 = list()
    # Extract the scan_info were the object_type is scan from assessor_info
    scan_info, assessor_info = extract_info(proc_info)

    # list of scans and assessors for the full project
    assessor_list = XnatUtils.list_project_assessors(xnat, options.project)
    assessor_list = filter_list(subjects, 'subject_label', assessor_list)

    # From scan:
    if scan_info:
        records1 = get_proc_scan_xnat(
            options, xnat, directory, subjects, scan_info, assessor_list)

    # From assessor:
    if assessor_info:
        records2 = get_proc_assr_xnat(
            options, xnat, directory, assessor_info, assessor_list)

    return records1 + records2


def get_proc_scan_xnat(options, xnat, directory, subjects, scan_info,
                       assessor_list):
    """
    Extract information from XNAT for the csv for processed data on scan
    """
    proc_records = list()

    # list of scans and assessors for the full project
    scan_list = XnatUtils.list_project_scans(xnat, options.project)

    # filter to keep only the scan we need and keep the assessor that we need
    # for the qc Scan
    scantypes = [v['proc_types'] for _, v in list(scan_info.items())]
    scan_list = filter_list(scantypes, 'type', scan_list)
    scan_list = filter_list(['usable'], 'quality', scan_list)
    scan_list = filter_list(subjects, 'subject_label', scan_list)
    # Assessor:
    proctypes = [v['proc_types'] for _, v in list(scan_info.items())]
    assessor_list = filter_list(proctypes, 'proctype', assessor_list)

    print('INFO: Downloading scans processed resources from XNAT')
    # For each scan, download the data and write the scan_records for the csv
    for scan in sorted(scan_list, key=lambda k: k['subject_label']):
        mess = """ subject/session/scan: {subj}/{sess}/{scan} found"""
        print(mess.format(subj=scan['subject_label'],
                          sess=scan['session_label'],
                          scan=scan['ID']))
        scan_dict = scan.copy()  # Copy the dict
        # qc outcome
        qc_status, qc_reason = get_qc_proc_scan(scan)
        scan_dict['qc_outcome'] = qc_status
        scan_dict['qc_fail_quest_reason'] = qc_reason
        for header, value in list(scan_info[scan['type']].items()):
            if header == 'derived_files_xnat':
                scan_dict['derived_files_xnat'] = download_scan_file(
                    xnat, directory, scan, value.split(','), header)
            else:
                scan_dict[header] = value

        # Add label for jobname:
        label = '-x-'.join([scan['subject_label'], scan['session_label'],
                            scan['ID']])
        scan_dict['label'] = label + '-x-skull_stripping'
        proc_records.append(scan_dict)

    return proc_records


def get_proc_assr_xnat(options, xnat, directory, assessor_info, assessor_list):
    """ Extract information from XNAT for the csv for processed data on
    assessor
    """
    proc_records = list()

    # filter to keep only the scan we need and keep the assessor that we need
    # for the qc
    proctypes = [v['proc_types'] for _, v in list(assessor_info.items())]
    assessor_list = filter_list(proctypes, 'proctype', assessor_list)
    assessor_list = filter_list(['COMPLETE', 'READY_TO_COMPLETE'],
                                'procstatus', assessor_list)

    print('INFO: Downloading assessors resources from XNAT')
    # For each scan, download the data and write the scan_records for the csv
    for assessor in sorted(assessor_list, key=lambda k: k['subject_label']):
        if XnatUtils.is_bad_qa(assessor['qcstatus']) in [0, 1]:
            # keep only the data that finished
            mess = """ assessor: {assessor} found"""
            print(mess.format(assessor=assessor['label']))
            assessor_dict = assessor.copy()  # Copy the dict
            # qc outcome
            qc_status, qc_reason = get_qc_assessor(assessor)
            assessor_dict['qc_outcome'] = qc_status
            assessor_dict['qc_fail_quest_reason'] = qc_reason

            assr_dict = assessor_info[assessor['proctype']]
            for header, value in list(assr_dict.items()):
                if header == 'derived_files_xnat':
                    if value == 'all':  # remove the metric_files_xnat
                        resources = XnatUtils.list_assessor_out_resources(
                            xnat, assessor['project_id'],
                            assessor['subject_label'],
                            assessor['session_label'],
                            assessor['label'])
                        resources = [res['label'] for res in resources
                                     if res['label'] not in ['OUTLOG', 'PBS', assr_dict['metric_files_xnat']]]
                    else:
                        resources = value.split(',')
                    assessor_dict[header] = download_assessor_file(
                        xnat, directory, assessor, resources, header)
                elif header == 'metric_files_xnat' and value:
                    assessor_dict[header] = download_assessor_file(
                        xnat, directory, assessor, value.split(','), header)
                else:
                    assessor_dict[header] = value
            proc_records.append(assessor_dict)

    return proc_records


def download_scan_file(xnat, directory, scan, resources, fname):
    """Download files from Scan determine by resources:
        xnat: interface object to xnat
        directory: root directory
        assessor: assessor dictionary
        resources: label of the resource to download
        fname: folder name
        return fpath (zip archive or file if only one)
    """
    fpaths = list()
    fpath = ''
    # String length to substract from the fpath
    # (NDAR want the path to start from the directory you submit data from and
    #  not full path)
    string_len = len(directory)
    foldername = '-x-'.join([scan['subject_label'], scan['session_label'],
                             scan['ID']])
    res_path = os.path.join(directory, foldername, fname)
    if glob.glob(os.path.join(res_path, '*')):
        fpath = glob.glob(os.path.join(res_path, '*'))[0][string_len:]
    else:
        # No file, download the resource data
        if not os.path.exists(res_path):
            os.makedirs(res_path)
        # Select the scan:
        scan_obj = XnatUtils.get_full_object(xnat, scan)
        for resource in resources:
            if resource in ['bval', 'bvec']:
                if scan_obj.resource(resource.lower()).exists():
                    res_obj = scan_obj.resource(resource.lower())
                    fpath = XnatUtils.download_biggest_file_from_obj(
                        res_path, res_obj)
                    fpaths.append(fpath)
                elif scan_obj.resource(resource.upper()).exists():
                    res_obj = scan_obj.resource(resource.upper())
                    fpath = XnatUtils.download_biggest_file_from_obj(
                        res_path, res_obj)
                    fpaths.append(fpath)
            elif scan_obj.resource(resource).exists():
                res_obj = scan_obj.resource(resource)
                fpath = XnatUtils.download_biggest_file_from_obj(
                    res_path, res_obj)
                fpaths.append(fpath)
        if len(fpaths) > 1:
            fpath = zipping_resource(res_path, fname)
        elif len(fpaths) == 1:
            fpath = fpaths[0]
        else:
            msg = "Warning: no file downloaded for {} on {}"
            print(msg.format(','.join(resources), foldername))
    return fpath


def download_assessor_file(xnat, directory, assessor, resources, fname):
    """
    Download files from Assessor determine by resources:

    xnat: interface object to xnat
    directory: root directory
    assessor: assessor dictionary
    resources: label of the resource to download
    fname: folder name
    return fpath (zip archive or file if only one)
    """
    fpaths = list()
    fpath = ''
    string_len = len(directory)
    res_path = os.path.join(directory, assessor['label'], fname)
    if glob.glob(os.path.join(res_path, '*')):
        fpath = glob.glob(os.path.join(res_path, '*'))[0][string_len:]
    else:
        if not os.path.exists(res_path):
            os.makedirs(res_path)
        assessor_obj = XnatUtils.get_full_object(xnat, assessor)
        for resource in resources:
            if resource:
                res_obj = assessor_obj.out_resource(resource)
                fpaths.extend(
                    XnatUtils.download_files_from_obj(res_path, res_obj))
            else:
                pass
        if len(fpaths) > 1:
            fpath = zipping_resource(res_path, fname)
        elif len(fpaths) == 1:
            fpath = fpaths[0]
        else:
            print("Warning: no file downloaded for {} on {}".format(
                ','.join(resources), assessor['label']))
    return fpath


def zipping_resource(folder, fname):
    """ Zip the folder given with the name fname.zip """
    initdir = os.getcwd()
    # Zip all the files in the directory
    os.chdir(folder)
    zip_cmd = """zip -r {}.zip * > /dev/null""".format(fname)
    os.system(zip_cmd)
    # return to the initial directory:
    os.chdir(initdir)
    # Remove files from res_path that are not the zip file:
    for fn in os.listdir(folder):
        if fn != fname + '.zip':
            if os.path.isdir(os.path.join(folder, fn)):
                shutil.rmtree(os.path.join(folder, fn))
            else:
                os.remove(os.path.join(folder, fn))
    os.chdir(initdir)
    return os.path.join(folder, fname + '.zip')


def check_options(options):
    """
    Method to check options given to executables

    :param options: first output of OptParser.parse_args()
    :return: True if options are fine, False otherwise
    """
    # Checked argument values if not:
    if options.assessorinfo and options.scaninfo:
        print("OPTION WARNING: you gave the scan csv and the assessor csv. \
Only one at a time.")
        return False
    elif not options.assessorinfo and not options.scaninfo:
        print("OPTION WARNING: no scan csv or assessor csv given.")
        return False
    if options.subjectinfo and \
       not os.path.exists(os.path.abspath(options.subjectinfo)):
        print("OPTION ERROR: the CSV file for the subjects information {} \
does not exist.".format(options.subjectinfo))
        return False
    if options.scaninfo and \
       not os.path.exists(os.path.abspath(options.scaninfo)):
        print("OPTION ERROR: the CSV file for the scans information {} \
does not exist.".format(options.scaninfo))
        return False
    if options.assessorinfo and \
       not os.path.exists(os.path.abspath(options.assessorinfo)):
        print("OPTION ERROR: the CSV file for the scans information {} \
does not exist.".format(options.assessorinfo))
        return False
    if options.directory and \
       not os.path.exists(os.path.dirname(os.path.abspath(options.directory))):
        msg = 'OPTION ERROR: the directory options: {} can not be created \
because {} does not exist.'
        print(msg.format(options.directory, os.path.dirname(
            os.path.abspath(options.directory))))
        return False
    elif not options.directory:
        print("OPTION ERROR: the directory option wasn't specified.")
        return False
    if not options.project:
        print('OPTION ERROR: the XNAT project options has not been set.')
        return False
    else:
        # Connection to Xnat
        try:
            xnat = XnatUtils.get_interface()
            # check access
            proj = xnat.select('/project/{}'.format(options.project))
            if not proj.exists():
                print('OPTION ERROR: XNAT Project given {} does not exist on \
XNAT.'.format(options.project))
                return False
            else:
                if not XnatUtils.list_subjects(xnat, options.project):
                    msg = "OPTION ERROR: You don't access to the project: {}."
                    print(msg.format(options.project))
                    return False
        finally:
            xnat.disconnect()

    return True


def Main_display(parser):
    """
    Main display of the executables before any process

    :param parser: OptionParser Object
    :return: None
    """
    (options, _) = parser.parse_args()
    print('################################################################')
    print('#                           XNATNDAR                           #')
    print('#                                                              #')
    print('# Developed by the masiLab Vanderbilt University, TN, USA.     #')
    print('# If issues, please start a thread here:                       #')
    print('# https://groups.google.com/forum/#!forum/vuiis-cci            #')
    print('#                                                              #')
    print('# Function:                                                    #')
    print('#     prepare a directory for NDAR submission using image03    #')
    print('#     template (download data and generate csv)                #')
    print('#                                                              #')
    print('# Parameters :                                                 #')
    if options == {'directory': None, 'project': None,
                   'scaninfo': None, 'subjectinfo': None,
                   'continu': False}:
        print('#     No Arguments given                                      \
 #')
        print('#     Use "XnatNDAR -h" to see the options                    \
 #')
        print('##############################################################\
##')
        parser.print_help()
        sys.exit()
    else:
        if options.directory:
            print('#     %*s -> %*s#' % (
                -20, 'Submission Folder',
                -33, get_proper_str(options.directory, True)))
        if options.project:
            print('#     %*s -> %*s#' % (
                -20, 'XNAT Project', -33, get_proper_str(options.project)))
        if options.scaninfo:
            print('#     %*s -> %*s#' % (
                -20, 'CSV scan', -33, get_proper_str(options.scaninfo, True)))
        if options.assessorinfo:
            print('#     %*s -> %*s#' % (
                -20, 'CSV assessor',
                -33, get_proper_str(options.assessorinfo, True)))
        if options.subjectinfo:
            print('#     %*s -> %*s#' % (
                -20, 'CSV subject',
                -33, get_proper_str(options.subjectinfo, True)))
        if options.continu:
            print('#     %*s -> %*s#' % (-20, 'Mode Continue', -33, 'on'))
        print('##############################################################\
##')


def get_proper_str(str_option, end=False):
    """
    Method to shorten a string into the proper size for display

    :param str_option: string to shorten
    :param end: keep the end of the string visible (default beginning)
    :return: shortened string
    """
    if len(str_option) > 32:
        if end:
            return '...' + str_option[-29:]
        else:
            return str_option[:29] + '...'
    else:
        return str_option


def parse_args():
    """
    Method to parse arguments base on OptionParser

    :return: parser object
    """
    from optparse import OptionParser
    usage = "usage: %prog [options] \nWhat is the script doing : Generate a \
CSV file for NDAR submission for the template image_03 only."
    parser = OptionParser(usage=usage)
    # Submission folder:
    parser.add_option("-d", "--directory", dest="directory", default=None,
                      help="Directory to store the data for submission. \
NDAR_submission.csv will be created in it.", metavar="DIR")
    # XNAT information
    parser.add_option("-p", "--project", dest="project", default=None,
                      help="Project ID on Xnat", metavar="PROJECT_ID")
    # Outside information
    parser.add_option("-s", "--scaninfo", dest="scaninfo", default=None,
                      help="CSV file with the following header: xnat_scantype\
,xnat_series_description,assessor_type_qc,ndar_scantype,image_num_dimensions,\
slice_acquisition,pulse_sequence,citation,experiment_id,image_file,\
image_thumbnail_file,data_file2,data_file2_definition. \
See script header for more information and an example.", metavar="CSV")
    parser.add_option("-a", "--assessorinfo", dest="assessorinfo",
                      default=None,
                      help="CSV file with the following header: proc_types,\
inputs,metric_files_xnat,derived_files_xnat,pipeline,pipeline_script,\
pipeline_tools,pipeline_version. See script header for more information \
and an example.", metavar="CSV")
    parser.add_option("-S", "--subjectinfo", dest="subjectinfo", default=None,
                      help="CSV file with the following header \
xnat_subject_id,GUID,study_subject_id,interview_date,interview_age,gender",
                      metavar="CSV")
    # options
    parser.add_option("-c", "--continue", dest="continu", action="store_true",
                      default=False,
                      help="If the script stopped, use continue to restart \
the script where it stopped.", metavar="FILEPATH")
    return parser


if __name__ == '__main__':
    """ Main Function """
    parser = parse_args()
    (options, args) = parser.parse_args()
    #############################
    # Main display:
    Main_display(parser)
    # check options:
    run = check_options(options)
    #############################

    #############################
    # RUN                       #
    #############################
    if run:
        directory = os.path.abspath(options.directory)
        # Output file:
        outputcsv = os.path.join(directory, 'NDAR_submission.csv')
        # Read from CSV files
        subject_records = read_csv(options.subjectinfo, DEFAULT_SUBJECT_HEADER)

        # Print number of object from CSV
        print('INFO: Number of Subject found in the csv:')
        print('---------------------------')
        print('| %*s : %*s |' % (
            -10, 'Subjects', -10, str(len(subject_records))))
        print('---------------------------')

        # Get Xnat info and download files:
        print('INFO: Querying XNAT project {} to download data.'.format(
            options.project))
        with XnatUtils.get_interface() as xnat:
            if options.scaninfo:
                records = get_scan_xnat(
                    options, xnat, directory, list(subject_records.keys()))
            elif options.assessorinfo:
                records = get_processed_data_xnat(
                    options, xnat, directory, list(subject_records.keys()))

        if not records:
            print('WARNING: No record found on XNAT. Please check the inputs.')
            sys.exit()

        # Print number of scans from xnat
        print('INFO: Number of Subject found on XNAT for NDAR submission and \
the number of records (one record per scan/processed data):')
        print('---------------------------')
        print('| %*s : %*s |' % (-10, 'Subjects',
                                 -10, str(len(set([record['subject_label']
                                                   for record in records])))))
        print('| %*s : %*s |' % (-10, 'Records', -10, str(len(records))))
        print('---------------------------')

        # Open the CSV file NDAR_submission.csv:
        with open(outputcsv, 'wb') as csvfile:
            spamwriter = csv.writer(csvfile, delimiter=',')
            if options.scaninfo:
                write_csv(outputcsv, SCAN_HEADER, SCAN_ORDERED_KEYS, records,
                          get_scan_row, options.continu)
            elif options.assessorinfo:
                write_csv(outputcsv, ASSESSOR_HEADER, ASSESSOR_ORDERED_KEYS,
                          records, get_assessor_row, options.continu)
