@4 Port USB Hub: Hilfe, was mach ich falsch? Will die Excel-Ausgabe für die Tanelorn-Auswertungen implementieren.
#!/usr/bin/python
# coding=utf-8
import urllib.request
import xlsxwriter
from statistics import StatisticsError, mean
from bs4 import BeautifulSoup
from collections import namedtuple, OrderedDict
from operator import attrgetter
from multiprocessing.dummy import Pool as ThreadPool
# Collection of Thread IDs in several categories
Produktthreads = OrderedDict([
('Spielhilfen', [
100244 ,
99039 ,
98315 ,
103886
]),
('Abenteuer', [
97188 ,
98757 ,
100657 ,
103971
])
])
# maintain anthologies separately
Anthologien = OrderedDict([
('Cthulhu - Ars Mathematica',[
102159, 102160, 102158]),
('Cthulhu - Dreissig',[
101501, 101503, 101504, 101502]),
('Cthulhu - The Final Revelation',[
97284, 97285, 97286, 97287]),
('Cthulhu - Die Goldenen Hände Suc´naaths',[
98758, 98757, 98759]),
('Shadowrun - Licht aus der Asche',[
96028, 96027, 96026])
])
# Add anthologies to collection to avoid duplicates
for Anthologie in Anthologien:
for threadid in Anthologien[Anthologie]:
if threadid not in Produktthreads['Abenteuer']:
Produktthreads['Abenteuer'].append(threadid)
# URL of a thread (%d will be thread_id)
baseurl = "https://www.tanelorn.net/index.php?topic=%d.0"
# Number of parallel threads (should be equal to number of CPU cores)
concurrent_parses = 4
def bbcode(tag, string, value=None):
"""Return a text(string) enclosed by the bbcode tags"""
if value:
return'[' + tag + '=' + value + ']' + string + '[/' + tag + ']'
else:
return'[' + tag + ']' + string + '[/' + tag + ']'
def bbcodeurl(urlstring, urlname):
"""Return an bbcode url format for given url and description"""
return bbcode('url', urlname, urlstring)
def bbbold(text):
"""Return the text with a bbcode bold tag"""
return bbcode(tag='b', string=text)
def bbtt(text):
"""Return the text with a bbcode tt tag"""
return bbcode(tag='tt', string=text)
def generate_xlsx(bewertungs_threads):
# Create a workbook and add a worksheet.
workbook = xlsxwriter.Workbook('Bewertungen.xlsx')
worksheet = workbook.add_worksheet()
# Start from the first cell. Rows and columns are zero indexed.
row = 0
col = 0
# Iterate over the data and write it out row by row.
for index, element in bewertungsthreads:
worksheet.write(row, col, index + 1)
worksheet.write(row, col + 1, element.Durchschnitt)
worksheet.write(row, col + 2, element.Stimmen)
worksheet.write(row, col + 3, element.url)
worksheet.write(row, col + 4, element.name)
row += 1
workbook.close()
class bbtable():
"""creates the frame of a bbcode table"""
def __init__(self, rows):
"""needs the rows as input for this table"""
self.elements = rows
def tablify(self, rows):
"""adds start and end tags for tables"""
return str('[table]\r\n' + rows + '[/table]')
def __str__(self):
"""prints table in bbcode format"""
return(self.tablify(''.join(str(row) for row in self.elements)))
class tablerow(bbtable):
"""creates a bbcode table row with correct tags"""
def cellify(self, rowfield):
"""encloses cells with correct tags"""
return str('[td]' + str(rowfield) + '[/td]')
def rowify(self, cells):
"""encloses rows with the correct tags"""
return str('[tr]' + str(cells) + '[/tr]\r\n')
def __str__(self):
"""adds cell and row tags to elements"""
return(self.rowify(''.join(self.cellify(field) for field in self.elements)))
class tableheaderrow(tablerow):
"""adds a header row"""
def cellify(self, rowfield):
return str('[td]' + bbbold(rowfield) + bbtt(' ') + '[/td]')
class ProduktParser():
def __init__(self, Produktthreads, Produkt = namedtuple('Produkt', 'name id url Stimmen Durchschnitt'), Produkte = [], Anthologien = [], baseurl = baseurl):
"""set base properties: URLs, thread ids, format"""
self.Produkt = Produkt
self.Produkte = Produkte
self.baseurl = baseurl
self.Produktthreads = Produktthreads
self.Anthologien = Anthologien
self.bewertungen = set(
[item for sublist in self.Produktthreads.values() for item in sublist])
self.pool = ThreadPool(concurrent_parses)
self.pool.map(self.getProdukt, self.bewertungen)
self.getAnthologie()
def getProdukt(self, threadid):
"""collect information for selected thread id"""
url = self.baseurl % threadid
page = urllib.request.urlopen(url)
soup = BeautifulSoup(page.read(), "html.parser")
Produktname = soup.find('title').string.split('/')[0].strip()
polls = soup.find('dl', {'class': 'options'})
options = polls.findAll('dt', {'class': 'middletext'})
votes = polls.findAll('span', {'class': 'percentage'})
ergebnis = dict(zip([[int(s) for s in option.string.replace("(","").split() if s.isdigit()][0] for option in options], [int(vote.string.split(' ')[0]) for vote in votes]))
einzelvotes = [
item for sublist in [[k] * v for k, v in ergebnis.items()] for item in sublist]
try:
durchschnitt = str(round(mean(einzelvotes), 2))
stimmen = len(einzelvotes)
except (ZeroDivisionError, StatisticsError) as e:
durchschnitt = '0 / No votes yet'
stimmen = 0
self.Produkte.append(
self.Produkt(Produktname, threadid, url, stimmen, durchschnitt))
def getAnthologie(self):
for Anthologie in self.Anthologien:
Anthologiedurchschnittagg = 0
Anthologiestimmen = 0
for Spielhilfe in self.Produkte:
if Spielhilfe.id in self.Anthologien[Anthologie]:
if Spielhilfe.Durchschnitt != '0 / No votes yet':
Anthologiestimmen += Spielhilfe.Stimmen
Anthologiedurchschnittagg += Spielhilfe.Stimmen * float(Spielhilfe.Durchschnitt)
if Anthologiestimmen == 0:
Anthologiedurchschnitt = '0 / No votes yet'
else:
Anthologiedurchschnitt = str(round(Anthologiedurchschnittagg/Anthologiestimmen, 2))
self.Produkte.append(
self.Produkt(Anthologie, 0, 0, Anthologiestimmen, Anthologiedurchschnitt))
def generateTable(self, bewertungsthreads):
""""generate a table for the threads"""
return bbtable([tableheaderrow(['Platz', 'Bewertung', 'Stimmen', 'Produkt'])]
+ [tablerow([index + 1, element.Durchschnitt, element.Stimmen, bbcodeurl(element.url, element.name)])
for index, element in enumerate(sorted(bewertungsthreads, key=attrgetter('Durchschnitt'), reverse=True))])
def printProdukte(self):
""""print the table"""
for key, value in self.Produktthreads.items():
print('\r\n' + bbbold(key))
print(self.generateTable(
[Spielhilfe for Spielhilfe in self.Produkte if Spielhilfe.id in value]))
print('\r\n' + bbbold("Anthologien"))
print(self.generateTable(
[Spielhilfe for Spielhilfe in self.Produkte if Spielhilfe.name in [Anthologie for Anthologie in Anthologien]]))
if __name__ == '__main__':
TanelornParser = ProduktParser(Produktthreads=Produktthreads, Anthologien=Anthologien)
generate_xlsx(TanelornParser.get_all())
Traceback (most recent call last):
File "D:/RPG/tanelornbewertungen-xls.py", line 219, in <module>
generate_xlsx(TanelornParser.get_all())
AttributeError: 'ProduktParser' object has no attribute 'get_all'
