Config API Batch Run
Notes
-
Changes need only be made for Cells 4 and 5
-
Please follow comments to alter filenames, column names and filepath
-
The commented out section at the bottom of the notebook runs each row individually. It can be referenced for debugging
# dependencies
import json
import pandas as pd
import numpy as np
import requests
import csv
import os
import time
Helper Functions
1. A general function that takes a dictionary element and returns a flattened dictionary
# No Changes needed
def flatten_anything(dic, prefix = '', out_dic = {}):
if isinstance(dic, dict):
for key in dic.keys():
flatten_anything(dic[key], prefix + "." + key, out_dic)
elif isinstance(dic, list):
for i, elem in enumerate(dic):
flatten_anything(elem, prefix + "[" + str(i) + "]", out_dic)
else:
out_dic[prefix] = dic
return out_dic
2. Function to write files to a folder name passed
# No Changes needed
def write_to_file(row, source_name, path):
filename = path + source_name + ".csv"
write = not os.path.exists(filename)
with open(filename, 'a', newline='') as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=list(row.keys()))
if write:
writer.writeheader()
writer.writerow(row)
3. Function to Flatten provider level results
Inputs: API output Per Provider
Outputs: Dictionary element with Flattened Dict
# No changes needed
def flatten_this(ress, un_id = 'No ID Given'):
res = json.loads(ress)
final_dict = {}
final_dict['refine'] = flatten_anything(res['refine'], 'Refined_Attributes')
for key in res['output'].keys():
if 'data' in res['output'][key].keys():
final_dict[key] = flatten_anything(res['output'][key]['data'], key, {})
else:
final_dict[key] = {}
print ("\n", key, res['output'][key], "\n")
return final_dict
4. Function: Process a row of data
Inputs: Input row for input file
Outputs: A Dictionary element with flattened data for each provider
See API Documentation for instructions to generate API Key and find your channel ID
# No changes needed
def flatten_this(ress, un_id = 'No ID Given'):
res = json.loads(ress)
final_dict = {}
final_dict['refine'] = flatten_anything(res['refine'], 'Refined_Attributes')
for key in res['output'].keys():
if 'data' in res['output'][key].keys():
final_dict[key] = flatten_anything(res['output'][key]['data'], key, {})
else:
final_dict[key] = {}
print ("\n", key, res['output'][key], "\n")
return final_dict
Run The input file
with open('inputs.csv', newline='') as csvfile: # Replace with Input File name
reader = csv.DictReader(csvfile)
for i,row in enumerate(reader):
print (i, row['unique_id']) # Uncomment to monitor results
# Returns Dict element with flattened json for each provider results
res = process_row(row) # Remove 'True' from process_row function above if we want each provider result to be written to separate files
for key in res.keys(): #Loop through each provider
if len(res[key]) > 0:
write_to_file({**row, **res[key]}, key, "Results/") # Change to required file path
# example format of how an input file csv will be read in process_row function
# row = {
# 'first_name': 'John',
# 'last_name': 'Doe',
# 'address': '1234 Avenue City ST, 00000'
# }
# process_row(row)