Skip to content

Instantly share code, notes, and snippets.

@automation-stack
Last active September 24, 2017 02:57
Show Gist options
  • Select an option

  • Save automation-stack/8d0502077ebe1dbe804465b784c17519 to your computer and use it in GitHub Desktop.

Select an option

Save automation-stack/8d0502077ebe1dbe804465b784c17519 to your computer and use it in GitHub Desktop.
import aiohttp
import uvloop
from time import time
from asyncio import get_event_loop, ensure_future, set_event_loop
# Example for asynchronous library `aiohttp` (using only main thread)
set_event_loop(uvloop.new_event_loop())
async def sub(url):
async with aiohttp.ClientSession() as session:
async with session.get(url) as resp:
# print(resp.status)
return resp
async def run():
t1 = time()
futures = [
ensure_future(sub(url))
for url in [
f'http://httpbin.org/get?{i}'
for i in range(1, 100)
]
]
t2 = time()
results = [await future for future in futures]
print(results)
t3 = time()
print(t2 - t1, t3 - t2)
loop = get_event_loop()
loop.run_until_complete(run())
# Example for synchronous library `requests` (using ThreadExecutor)
import requests
import uvloop
from time import time
from functools import partial
from multiprocessing import cpu_count
from concurrent.futures import ThreadPoolExecutor
from asyncio import ensure_future, set_event_loop
class Requests(object):
def __init__(self):
self.executor = ThreadPoolExecutor(max_workers=cpu_count() * 10)
self.loop = uvloop.new_event_loop()
self.loop.set_default_executor(self.executor)
set_event_loop(self.loop)
self.futures = []
def __getattr__(self, item):
if hasattr(requests, item):
def wrapper(*args, **kwargs):
return self.add(getattr(requests, item), *args, **kwargs)
return wrapper
def add(self, callable_object, *args, **kwargs):
self.futures.append(self.loop.run_in_executor(
None, partial(callable_object, *args, **kwargs)
))
async def wait(self):
return [await future for future in self.futures]
@property
def results(self):
return self.loop.run_until_complete(self.wait())
t1 = time()
rq = Requests()
for i in range(1, 100):
rq.post(f'http://httpbin.org/post?{i}', data={'w': 2})
t2 = time()
# Get all requests results (blocking call)
results = rq.results
t3 = time()
print(results)
print(t2 - t1, t3 - t2)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment