Browse Source

first commit

master
Colat 6 months ago
commit
629b513f47
  1. 135
      .gitignore
  2. 41
      README.md
  3. 0
      __init.py__
  4. 31
      banner.txt
  5. 8
      config-example.py
  6. 57
      models.py
  7. 55
      parser.py
  8. 95
      run.py
  9. 2
      setup.py
  10. 59
      updater.py

135
.gitignore

@ -0,0 +1,135 @@
usurpa.pdf
# Configuration file, check config-example.py
config.py
# For pycharm ide
.idea
# Created by https://www.gitignore.io/api/python
# Edit at https://www.gitignore.io/?templates=python
### Python ###
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
pip-wheel-metadata/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
.hypothesis/
.pytest_cache/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
.python-version
# celery beat schedule file
celerybeat-schedule
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
### Python Patch ###
.venv/
# End of https://www.gitignore.io/api/python

41
README.md

@ -0,0 +1,41 @@
MMMMMMMMMMMMMMMMMMMMMMMMMWN0dc,.. ..,cx0NMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM
MMMMMMMMMMMMMMMMMMMMMWNOdc'. .,cdONMMMMMMMMMMMMMMMMMMMMMMMMMMM
MMMMMMMMMMMMMMMMMMMXkc' .'ckXMMMMMMMMMMMMWNXK0OkdoloO
MMMMMMMMMMMMMMMMW0l' 'l0WMMMWKdlc:,'... cX
MMMMMMMMMMMMMMWO:. .,:ldxkOOOkkxdoc;.. .c0WMMXl. cXM
MMMMMMMMMMMMMKc. .;oOXWMMMMMMMMMMMMMMWN0dc. .lKNOc. cXMM
MMMMMMMMMMMWk' ,dKWMMMMMMMMMMMMMMMMMMMMMMWXk:. .' cXMMM
MMMMMMMMMMNo. ,kNMMMMMMMMMMMMMMMMMMMMMMMMMMMMW0c. cXMMMM
MMMMMMMMMNl. .dNMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMWx. .;dx;.cXMMMMM
MMMMMMMMWd. ,OWMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMW0o, .l0WMMN0XMMMMMM
MMMMMMMMO. ,0MMMMMMMMMMMMMMMMWXXWMMMMMMMMMMWXx;. 'OMMMMMMMMMMMMM
MMMMMMMNc .OMMMMMMMMMMMMMMWXx:..lKWMMMMMMNkc. lNMMMMMMMMMMMM
MMMMMMMO. oWMMMMMMMMMMMMNOc. .oOXWW0o, ';. '0MMMMMMMMMMMM
MMMMMMWd. .OMMMMMMMMMMN0o' 'c;. .:kXX: .xMMMMMMMMMMMM
MMMMMMWo ,KMMMMMMMWKd;. ,oKWMMWl dWMMMMMMMMMMM
MMMMMMWo ,KMMMMWXx:. .cONMMMMMWl dWMMMMMMMMMMM
MMMMMMWd. '0MMNOc. .cxx; .;xXWMMMMMMMNc .xMMMMMMMMMMMM
MMMMMMMO. .o0o, .:xXWMMNx, 'o0WMMMMMMMMMM0' '0MMMMMMMMMMMM
MMMMMMMNc .. .;dKWMMMMMMMXd:;ckNMMMMMMMMMMMMNl lNMMMMMMMMMMMM
MMMMMMMMO. .,d0WMMMMMMMMMMMMWWWMMMMMMMMMMMMMWx. '0MMMMMMMMMMMMM
MMMMMMMNx' ,o0WMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMWx. .xWMMMMMMMMMMMMM
MMMMW0o, ,kNMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXl. .dNMMMMMMMMMMMMMM
MWKd;. .lKWMMMMMMMMMMMMMMMMMMMMMMMMMMMMMNx' .dNMMMMMMMMMMMMMMM
k:. .. .cONMMMMMMMMMMMMMMMMMMMMMMMMWKd, ,OWMMMMMMMMMMMMMMMM
; .:xKKl. .,oOXWMMMMMMMMMMMMMMMMMN0x:. .oXMMMMMMMMMMMMMMMMMM
Xo. .;dKWMMMW0c. .;ldO0KXNNNXXK0Oxo:'. .c0WMMMMMMMMMMMMMMMMMMM
MWO:;o0WMMMMMMMMW0o' .......... 'oKWMMMMMMMMMMMMMMMMMMMMM
MMMWWMMMMMMMMMMMMMMNkl'. .,lkNMMMMMMMMMMMMMMMMMMMMMMMM
MMMMMMMMMMMMMMMMMMMMMMN0dc,. ..,cd0NMMMMMMMMMMMMMMMMMMMMMMMMMMM
MMMMMMMMMMMMMMMMMMMMMMMMMWN0xl;.. ..;lx0NMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM
Welcome to usurpa-api !
# Usurpa-api
This api is based on the popular, ancient and usable squat schedule:
https://usurpa.squat.net
The objective of the api is to modernize the work of usurpa mates to be more adapted to our times.

0
__init.py__

31
banner.txt

@ -0,0 +1,31 @@
MMMMMMMMMMMMMMMMMMMMMMMMMWN0dc,.. ..,cx0NMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM
MMMMMMMMMMMMMMMMMMMMMWNOdc'. .,cdONMMMMMMMMMMMMMMMMMMMMMMMMMMM
MMMMMMMMMMMMMMMMMMMXkc' .'ckXMMMMMMMMMMMMWNXK0OkdoloO
MMMMMMMMMMMMMMMMW0l' 'l0WMMMWKdlc:,'... cX
MMMMMMMMMMMMMMWO:. .,:ldxkOOOkkxdoc;.. .c0WMMXl. cXM
MMMMMMMMMMMMMKc. .;oOXWMMMMMMMMMMMMMMWN0dc. .lKNOc. cXMM
MMMMMMMMMMMWk' ,dKWMMMMMMMMMMMMMMMMMMMMMMWXk:. .' cXMMM
MMMMMMMMMMNo. ,kNMMMMMMMMMMMMMMMMMMMMMMMMMMMMW0c. cXMMMM
MMMMMMMMMNl. .dNMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMWx. .;dx;.cXMMMMM
MMMMMMMMWd. ,OWMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMW0o, .l0WMMN0XMMMMMM
MMMMMMMMO. ,0MMMMMMMMMMMMMMMMWXXWMMMMMMMMMMWXx;. 'OMMMMMMMMMMMMM
MMMMMMMNc .OMMMMMMMMMMMMMMWXx:..lKWMMMMMMNkc. lNMMMMMMMMMMMM
MMMMMMMO. oWMMMMMMMMMMMMNOc. .oOXWW0o, ';. '0MMMMMMMMMMMM
MMMMMMWd. .OMMMMMMMMMMN0o' 'c;. .:kXX: .xMMMMMMMMMMMM
MMMMMMWo ,KMMMMMMMWKd;. ,oKWMMWl dWMMMMMMMMMMM
MMMMMMWo ,KMMMMWXx:. .cONMMMMMWl dWMMMMMMMMMMM
MMMMMMWd. '0MMNOc. .cxx; .;xXWMMMMMMMNc .xMMMMMMMMMMMM
MMMMMMMO. .o0o, .:xXWMMNx, 'o0WMMMMMMMMMM0' '0MMMMMMMMMMMM
MMMMMMMNc .. .;dKWMMMMMMMXd:;ckNMMMMMMMMMMMMNl lNMMMMMMMMMMMM
MMMMMMMMO. .,d0WMMMMMMMMMMMMWWWMMMMMMMMMMMMMWx. '0MMMMMMMMMMMMM
MMMMMMMNx' ,o0WMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMWx. .xWMMMMMMMMMMMMM
MMMMW0o, ,kNMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXl. .dNMMMMMMMMMMMMMM
MWKd;. .lKWMMMMMMMMMMMMMMMMMMMMMMMMMMMMMNx' .dNMMMMMMMMMMMMMMM
k:. .. .cONMMMMMMMMMMMMMMMMMMMMMMMMWKd, ,OWMMMMMMMMMMMMMMMM
; .:xKKl. .,oOXWMMMMMMMMMMMMMMMMMN0x:. .oXMMMMMMMMMMMMMMMMMM
Xo. .;dKWMMMW0c. .;ldO0KXNNNXXK0Oxo:'. .c0WMMMMMMMMMMMMMMMMMMM
MWO:;o0WMMMMMMMMW0o' .......... 'oKWMMMMMMMMMMMMMMMMMMMMM
MMMWWMMMMMMMMMMMMMMNkl'. .,lkNMMMMMMMMMMMMMMMMMMMMMMMM
MMMMMMMMMMMMMMMMMMMMMMN0dc,. ..,cd0NMMMMMMMMMMMMMMMMMMMMMMMMMMM
MMMMMMMMMMMMMMMMMMMMMMMMMWN0xl;.. ..;lx0NMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM

8
config-example.py

@ -0,0 +1,8 @@
#!/usr/bin/python
# Name of the application
APP_BASE_NAME = 'usurpa-api'
# File to update
USURPA_URL = 'https://usurpa.squat.net/usurpa.pdf'
USURPA_DOWNLOAD_PATH = 'usurpa.pdf'

57
models.py

@ -0,0 +1,57 @@
import config
squatNameExceptions = config.NAME_EXCEPTIONS
zones = config.REGIONAL_ZONES
class Squat(object):
def __init__(self, name=None):
self.name = name
self.rows = []
def addRow(self, row):
self.rows.append(row)
@classmethod
def isSquatName(cls, name):
# TODO: the names that starts with caps and then minus (like L’ACABose) are not detected as squat name.
# Maybe is possible to think a rule like if the first two letters are capital?
if str.isupper(name) or squatNameExceptions in name:
return name
return False
class RegionalZone (object):
def __init__(self, name=None):
self.name = name
self.rows = []
self.squatList = []
def addRow(self, row):
"""
Add a row object [] into the sets of self.rows[]
:param row:
:return:
"""
self.rows.append(row)
# TODO: DRY this function is basically the same as parser.parseCSV
def generateSquats(self):
squat = None
for row in self.rows:
squatName = Squat.isSquatName(row[0])
if squatName:
squat = Squat(squatName)
self.squatList.append(squat)
if squat is not None:
squat.addRow(row)
# TODO: Try to do this without making loop
# Check if a row contains a zone name defined on config file
@classmethod
def checkIfIsZone(cls, row):
for zone in zones:
if zone in row:
return zone
return False

55
parser.py

@ -0,0 +1,55 @@
import tabula
import config
import csv
from models import RegionalZone
appName = config.APP_BASE_NAME
usurpaPdf = config.USURPA_DOWNLOAD_PATH
csvOut = config.USURPA_CSV
zoneList = []
def parseCSV(input=csvOut):
print("# Parsig", input)
print ("* Parsing areas")
with open(input, 'r') as csvFile:
reader = csv.reader(csvFile)
regionalZone = None
for row in reader:
zoneName = RegionalZone.checkIfIsZone(row)
# If is zone append it to the list of zones
if zoneName:
regionalZone = RegionalZone(zoneName)
zoneList.append(regionalZone)
if regionalZone is not None and not zoneName:
regionalZone.addRow(row)
csvFile.close()
print ("* Creating squat objects")
for zone in zoneList:
zone.generateSquats()
print("# Printing all")
x = 0
for zone in zoneList:
print("--",zone.name, "squats:", zone.squatList.__len__())
for squat in zone.squatList:
print("\t",squat.name)
for row in squat.rows:
print("\t -- ",row.__len__(), row)
x += 1
print("Total squats found", x)
print("Total areas found: ", zoneList.__len__())
def convertPdfToCsv(input=usurpaPdf, output=csvOut):
print("# Converting", input, "to", output)
df = tabula.convert_into(input, output, pages='all', multiple_tables=True, output_format="csv", stream=True)
print("Done.")
return output

95
run.py

@ -0,0 +1,95 @@
import sys, getopt, os, time
import updater, config, parser
appName = config.APP_BASE_NAME
def fileExists(file):
return os.path.isfile(file)
def updateUsurpa(output):
if output is None:
updater.updateUsurpa()
else:
updater.updateUsurpa(output)
def convertUsurpa(input=None, output=None):
if input is None : input = config.USURPA_DOWNLOAD_PATH
if output is None : output = config.USURPA_CSV
if input is None or not fileExists(input):
print ("Error: converting usurpa can't find ", input, 'or', output)
elif output is None:
print("Error:", output, "is None")
else:
parser.convertPdfToCsv(input=input, output=output)
def parseUsurpa(input=None):
if input is None : input = config.USURPA_CSV
if fileExists(input):
parser.parseCSV(input=input)
else:
print ("Error: can't find", input)
# Main
def main(argv):
try:
def printBanner():
f = open('banner.txt', 'r')
file_contents = f.read()
print( file_contents, end="", flush=True)
print ("Welcome to ", appName, "!\n", end="", flush=True)
def printHelp ():
print("Help me to squat!!")
print("\nExample:\n\tpython run.py -u -c -p\n\nOptions:")
print("\t-h Print this help")
print("\t-i Specify input file")
print("\t-o Specify output file")
print("\t-u Update", config.USURPA_DOWNLOAD_PATH, "from", config.USURPA_URL,
"\n\t Specify output with -o")
print("\t-c Convert", config.USURPA_DOWNLOAD_PATH, "to", config.USURPA_CSV,
"\n\t Specify -i -o parameters to especify input and output ")
print("\t-p Parse", config.USURPA_CSV,". Specify -i to specify the input file")
print("\t-❤️ Retrieve list of all empty houses to squat")
printBanner()
if argv.__len__() == 0:
printHelp()
try:
opts, args = getopt.getopt(argv, "hucpi:o:", "parse=") # Letters recognized. If a letter has argument follow it with a colon :, like p: fill be '-p arg'
except getopt.GetoptError:
printHelp()
sys.exit(2)
input = None
output = None
for opt, arg in opts:
if opt == '-h':
printHelp()
# sys.exit(0)
break
elif opt in ("-i"):
input = arg
elif opt in ("-o"):
output = arg
elif opt in ("-u"):
updateUsurpa(output)
elif opt in ("-c"):
convertUsurpa(input=input, output=output)
elif opt in ("-p"):
parseUsurpa(input=input)
# elif opt in ("--parse"):
# file = None
# if arg != "":
# file=arg
# parseUsurpa(file=file)
elif opt in ("-❤️"):
printHelp()
except:
raise BaseException("Error: failed to execute the program")
if __name__ == '__main__':
main(sys.argv[1:])

2
setup.py

@ -0,0 +1,2 @@
#!/usr/bin/python

59
updater.py

@ -0,0 +1,59 @@
import sys, getopt, os, time
from datetime import datetime
import config
import urllib.request
appName = config.APP_BASE_NAME
usurpaURL = config.USURPA_URL
usurpaDownloadPath = config.USURPA_DOWNLOAD_PATH
def isUsurpadownloaded():
return os.path.isfile(usurpaDownloadPath)
def getConnection(url = usurpaURL):
return urllib.request.urlopen(url, timeout=30)
# Get last modified attribure of remote url
def getRemoteLastModified(conn):
return datetime.strptime(conn.headers['last-modified'], '%a, %d %b %Y %H:%M:%S GMT')
# Get last modified of local file
def getLocalLastModified(file):
if not os.path.isfile(file):
return False
stat = os.stat(file)
try:
return datetime.fromtimestamp(stat.st_birthtime)
except AttributeError:
# We're probably on Linux. No easy way to get creation dates here,
# so we'll settle for when its content was last modified.
return datetime.fromtimestamp(stat.st_mtime)
def downloadUsurpa(url=usurpaURL, downloadPath = usurpaDownloadPath):
print ("- Start download of ", usurpaURL )
# Download the file
urllib.request.urlretrieve(url, downloadPath)
# Set modification date
t = time.mktime(getRemoteLastModified(getConnection()).timetuple())
os.utime(usurpaDownloadPath, (t, t))
def isUsurpaUpdated(remote, local):
if remote == local:
return True
return False
def updateUsurpa(output=usurpaDownloadPath):
print("# ", appName,"check for updates ", )
try:
if not isUsurpadownloaded:
print("- No ", appName," file found, proceed to download on\n", output)
downloadUsurpa(usurpaURL, downloadPath=output)
elif not isUsurpaUpdated(getRemoteLastModified(getConnection()), getLocalLastModified(output)):
print("- ", appName," is not updated, proceed to update: \n", output)
downloadUsurpa(usurpaURL, downloadPath=output)
print(appName, "updated succesfully")
print("Done.")
else:
print(appName, "was already updated")
except:
raise Exception("Error: Failed to update usurpa")
Loading…
Cancel
Save