Home Having trouble while fetching data from PYTHON script using AWS Athena-Boto3
Reply: 0

Having trouble while fetching data from PYTHON script using AWS Athena-Boto3

user44385
1#
user44385 Published in September 19, 2018, 11:40 am

I am trying to query the dataset present in s3 bucket, using Athena query via python script with help of boto3 functions.

I am using start_query_execution() to run my query. this is being executed perfectly, next to get results in my python script, so that I get access to the result of the query I am using the function get_query_results().

Now if I run these two function separately(one script after another) I get the data which is an output of Athena query. I want them to be written in a single script - something like, fetch data from s3 and start manipulating the output of query using python code.

Since the query is asyn in nature, i am using pool technique, where it waits till the Athena query is executed. But if i run the below codethe, the status show is running for the query.

I think I am doing some silly mistake as if I run them separately I get desired output. In short, I want to query the data present in s3 using Athena, then do some processing on this fetched data in python script, hence this approach. Please help

Here is the sample code

#!/usr/bin/env python3

import boto3
import time
from functools import partial
from multiprocessing.dummy import Pool
pool = Pool(processes=1)

# def async_function(name):
#     time.sleep(1)
#     return name
#
# def callback_function(name, age):
#     print(name, age)

def run_query(query, database, s3_output):
    client = boto3.client('athena')
    response = client.start_query_execution(
        QueryString=query,
        QueryExecutionContext={
            'Database': database
            },
        ResultConfiguration={
            'OutputLocation': s3_output,
            }
        )
    print('Execution ID: ' + response['QueryExecutionId'])
    return response
def show_res(res, q):
    client = boto3.client('athena')
    print("Executing query: %s" % (q))
    print('Execution ID: ' + res['QueryExecutionId'])
    # response = client.stop_query_execution(
    #     QueryExecutionId=res['QueryExecutionId']
    # )
    response = client.get_query_results(
        # QueryExecutionId='f3642735-d9d9-4246-ade4-7453eaed0717'
        QueryExecutionId=res['QueryExecutionId']
        )
    print("Executing query: %s" % (q))
    print('Execution ID: ' + res['QueryExecutionId'])
    print('rRespone:'.join(str(x) for x in response['ResultSet']['Rows']));
    return response

# for age, name in enumerate(['jack', 'jill', 'james']):
#     new_callback_function = partial(callback_function, age=age)
#     pool.apply_async(
#         async_function,
#         args=[name],
#         callback=new_callback_function
#     )

#Athena configuration
s3_input = 's3://dummy/'
s3_ouput = 's3://dummy/results/'
database = 'dummy'
table = 'dummy'

#Query definitions
query_1 = "SELECT * FROM %s.%s where sex = 'F';" % (database, table)
query_2 = "SELECT * FROM %s.%s where age > 30;" % (database, table)
#Execute all queries
queries = [ query_1 ]
for q in queries:
    print("Executing query: %s" % (q))
    new_callback_function = partial(show_res, q=q)
    pool.apply_async(
        run_query,
        args=[q, database, s3_ouput],
        callback=new_callback_function
    )

pool.close()
pool.join()
share|improve this question

active oldest votes

Your Answer

StackExchange.ifUsing("editor", function () { StackExchange.using("externalEditor", function () { StackExchange.using("snippets", function () { StackExchange.snippets.init(); }); }); }, "code-snippets"); StackExchange.ready(function() { var channelOptions = { tags: "".split(" "), id: "1" }; initTagRenderer("".split(" "), "".split(" "), channelOptions); StackExchange.using("externalEditor", function() { // Have to fire editor after snippets, if snippets enabled if (StackExchange.settings.snippets.snippetsEnabled) { StackExchange.using("snippets", function() { createEditor(); }); } else { createEditor(); } }); function createEditor() { StackExchange.prepareEditor({ heartbeatType: 'answer', convertImagesToLinks: true, noModals: false, showLowRepImageUploadWarning: true, reputationToPostImages: 10, bindNavPrevention: true, postfix: "", onDemand: true, discardSelector: ".discard-answer" ,immediatelyShowMarkdownHelp:true }); } });
 
StackExchange.ready( function () { StackExchange.openid.initPostLogin('.new-post-login', 'https%3a%2f%2fstackoverflow.com%2fquestions%2f49855087%2fhaving-trouble-while-fetching-data-from-python-script-using-aws-athena-boto3%23new-answer', 'question_page'); } );

By clicking "Post Your Answer", you acknowledge that you have read our updated terms of service, privacy policy and cookie policy, and that your continued use of the website is subject to these policies.

Browse other questions tagged python amazon-web-services asynchronous boto3 amazon-athena or ask your own question.

StackExchange.ready(function(){$.get('/posts/49855087/ivc/54f1');});
StackExchange.ready(function () { StackExchange.responsiveness.addSwitcher(); }) (function(i, s, o, g, r, a, m) { i['GoogleAnalyticsObject'] = r; i[r] = i[r] || function() { (i[r].q = i[r].q || []).push(arguments) }, i[r].l = 1 * new Date(); a = s.createElement(o), m = s.getElementsByTagName(o)[0]; a.async = 1; a.src = g; m.parentNode.insertBefore(a, m); })(window, document, 'script', 'https://www.google-analytics.com/analytics.js', 'ga'); StackExchange.ready(function () { StackExchange.ga.init({ sendTitles: true, tracker: window.ga, trackingCodes: [ 'UA-108242619-1' ] }); StackExchange.ga.setDimension('dimension2', '|python|amazon-web-services|asynchronous|boto3|amazon-athena|'); StackExchange.ga.setDimension('dimension3', 'Questions/Show'); StackExchange.ga.trackPageView(); }); /**/ var _qevents = _qevents || [], _comscore = _comscore || []; (function() { var ssl = 'https:' == document.location.protocol, s = document.getElementsByTagName('script')[0], qc = document.createElement('script'); qc.async = true; qc.src = (ssl ? 'https://secure' : 'http://edge') + '.quantserve.com/quant.js'; s.parentNode.insertBefore(qc, s); _qevents.push({ qacct: "p-c1rF4kxgLUzNc" }); /**/ var sc = document.createElement('script'); sc.async = true; sc.src = (ssl ? 'https://sb' : 'http://b') + '.scorecardresearch.com/beacon.js'; s.parentNode.insertBefore(sc, s); _comscore.push({ c1: "2", c2: "17440561" }); })();
You need to login account before you can post.

About| Privacy statement| Terms of Service| Advertising| Contact us| Help| Sitemap|
Processed in 0.44831 second(s) , Gzip On .

© 2016 Powered by mzan.com design MATCHINFO