edufour/MoodleScraper
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Changing moodle activity handling

Browse Source
This commit is contained in:
edufour
2024-01-27 01:02:41 +01:00
parent a738c6db39
commit 8e10323ac3
9 changed files with 320 additions and 247 deletions
Download Patch File Download Diff File Expand all files Collapse all files

103
main.py
View File

@@ -1,9 +1,6 @@
import requests
from bs4 import BeautifulSoup
from MoodleCourse import MoodleCourse
import os
import pathvalidate
from urllib.parse import unquote
from MoodleClasses.MoodleCourse import MoodleCourse
def get_moodle_auth_cookie(s: requests.Session, url: str) -> requests.Session:
@@ -27,84 +24,7 @@ def get_courses(s: requests.Session, url: str) -> list[MoodleCourse]:
return list(map(lambda x: MoodleCourse(x[0], x[1]), zip(course_names, course_urls)))
def download_resource(s, activity):
directory = os.path.join(activity.parent_dir, activity.name)
directory = pathvalidate.sanitize_filepath(directory)
file = os.path.join(directory, unquote(activity.url.split('/')[-1]))
file = pathvalidate.sanitize_filepath(file)
if args.ignore_extension is not None and str(file).split(".")[-1] in args.ignore_extension:
return True
if os.path.exists(os.path.join(file)):
return True
if not os.path.exists(directory):
os.makedirs(directory)
try:
with s.get(activity.url, stream=True) as r:
with open(file+".part", 'wb') as f:
for chunk in r.iter_content(chunk_size=8192):
f.write(chunk)
os.rename(file+".part", file)
if os.path.exists(file + ".part"):
os.remove(file + ".part")
except:
if os.path.exists(file+".part"):
os.remove(file+".part")
return False
finally:
return True
def download_folder(s, activity):
directory = os.path.join(activity.parent_dir, activity.name)
directory = pathvalidate.sanitize_filepath(directory)
try:
with s.get(activity.url, stream=True) as r:
file = os.path.join(directory, unquote(r.headers['Content-Disposition'].split('\'')[-1]))
file = pathvalidate.sanitize_filepath(file)
if os.path.exists(os.path.join(file)):
return True
if not os.path.exists(directory):
os.makedirs(directory)
with open(file + ".part", 'wb') as f:
for chunk in r.iter_content(chunk_size=8192):
f.write(chunk)
os.rename(file + ".part", file)
if os.path.exists(file + ".part"):
os.remove(file + ".part")
except:
if os.path.exists(file+".part"):
os.remove(file+".part")
return False
finally:
return True
def download_assign(s, activity):
r = s.get(activity.url)
soup = BeautifulSoup(r.text, 'html.parser')
for candidate in soup.find_all("div", {"class": "fileuploadsubmission"}):
link = candidate.find("a")
url = link["href"].split('?')[0]
with s.get(url, stream=True) as r:
directory = os.path.join(activity.parent_dir, activity.name)
directory = pathvalidate.sanitize_filepath(directory)
file = os.path.join(directory, unquote(link.text))
file = pathvalidate.sanitize_filepath(file)
if not os.path.exists(directory):
os.makedirs(directory)
if os.path.exists(os.path.join(file)):
return True
with open(file + ".part", 'wb') as f:
for chunk in r.iter_content(chunk_size=8192):
f.write(chunk)
os.rename(file + ".part", file)
if os.path.exists(file + ".part"):
os.remove(file + ".part")
return True
def download_year(target_url: str):
def download_year(target_url: str) -> None:
s = requests.Session()
s = get_moodle_auth_cookie(s, target_url)
courses = get_courses(s, target_url)
@@ -114,20 +34,11 @@ def download_year(target_url: str):
course.set_parent_dir(args.storage_dir, args.academic_year)
for section in course.sections:
for activity in section.activities:
if activity.type == "resource":
if activity.find_file(s):
if not download_resource(s, activity):
print("Could not download: ", activity.url)
else:
print("Could not find: ", activity.url)
elif activity.type == "folder":
if activity.find_folder(s):
if not download_folder(s, activity):
print("Could not download: ", activity.url)
else:
print("Could not find: ", activity.url)
elif activity.type == "assign":
download_assign(s, activity)
if activity.find(s):
if not activity.download(s, args.ignore_extension):
print(f"Could not download: {activity}")
else:
print(f"Could not find: {activity}")
if __name__ == "__main__":