notwa/respodns
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

rewrite IpReader iterator to accumulate IPs before yielding

Browse source 
still needs work
This commit is contained in:
Connor Olding 2020-09-04 13:36:59 +02:00
parent e6c080bf32
commit b26a4e7646
2 changed files with 19 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
respodns/dns.py
View file

@ -225,7 +225,7 @@ async def main(db, filepaths, checks, opts: Options):
for i, ip in enumerate(read_ips(f)):
first = i == 0
if opts.progress:
print(f"#{i}: {ip}", file=stderr)
print(f"#{i + 1}: {ip}", file=stderr)
stderr.flush()
if not first:
await sleep(opts.ip_wait)
@ -239,7 +239,7 @@ async def main(db, filepaths, checks, opts: Options):
for i, ip in enumerate(reader):
first = i == 0
if opts.progress:
print(f"#{i}/{reader.total}: {ip}", file=stderr)
print(f"#{i + 1}/{reader.total}: {ip}", file=stderr)
stderr.flush()
if not first:
await sleep(opts.ip_wait)

32
respodns/ip_util.py
View file

@ -52,27 +52,29 @@ class IpReader:
self.threads = []
self.total = 0
def running(self):
def is_running(self):
return any(thread.is_alive() for thread in self.threads)
def __iter__(self):
return self
def __next__(self):
# TODO: rewrite such that self.total is useful. (get as many at once)
from queue import Empty
from sys import stderr
while self.running() or not self.queue.empty():
try:
res = self.queue.get(block=True, timeout=1.0)
if res is not None:
self.total += 1
return res
except Empty:
from sys import stderr
print("blocking on IpReader", file=stderr)
def _next():
while self.is_running() or not self.queue.empty():
results = []
if self.queue.empty():
try:
results.append(self.queue.get(timeout=1.0))
except Empty:
print("blocking on IpReader", file=stderr)
else:
while not self.queue.empty():
results.append(self.queue.get())
self.total += len(results)
for res in results:
yield res
raise StopIteration
return _next()
def __enter__(self):
from threading import Thread