import time from globals import * from bs4 import BeautifulSoup __author__ = "Jochem van Dolder" __license__ = "MIT" __all__ = ["MudaeImage", "char_attr", "character"] ### --- MUDAE IMAGE CLASS --- ### tic = 0 class MudaeImage(): def __init__(self, number:str, title:str, src:str, color:str=DEFAULT_COLOR): self.number = int(number)-1 if number.isdigit() else -1 self.title = title.strip() if title else "" self.src = src self.color = color self.frame = 0 self.original_color = color self.loaded = False self.is_animated = False def set_color(self, color): self.color = color def set_frame(self, frame): if self.is_animated: self.pil = self.frames[frame] self.frame = frame def image(self): if self.loaded: return self.pil frames, durations = [], [] if self.src.startswith('http'): ret = get_frames_from_url(self.src) if ret: frames, durations = ret elif self.src[1:3] == ':/': ret = get_frames_from_file(self.src) if ret: frames, durations = ret if not frames: return if len(frames) > 1: self.is_animated = True self.frames, self.durations = frames, durations self.pil = self.frames[self.frame] else: self.is_animated = False self.pil = frames[0] self.loaded = True return self.pil def close(self): if self.is_animated: for frame in self.frames: frame.close() self.frames.clear() if hasattr(self, "pil"): self.pil.close() self.loaded = False def __repr__(self): out_str = '' key_list = self.__dict__.keys() banned = ('frame', 'durations', 'original_color', 'number', 'pil', 'loaded') if self.original_color == self.color: banned = (*banned,'color') keys = [key for key in key_list if (self.__dict__[key] and (key not in banned))] if keys: max_key_len = max([len(key[:6]) for key in keys]) else: return out_str for key in keys: if out_str: out_str+='\n' val = self.__dict__[key] key_val = len(val) if isinstance(val, list) else val out_str += f"├ {key[:6].replace('_',' ')}: {' '*(max_key_len-len(key))}{key_val}" return out_str def __bool__(self): return self.loaded ### --- CHARACTER CLASS --- ### class char_attr(): def __init__(self, item:str=''): self.item = item self.rank = 0 self.name = '' self.owner = '' self.info = '' self.note = '' self.tags = 0 self.type = '' self.keys = 0 self.soul_keys = 0 self.kakera = 0 self.spheres = 0 self.url = '' self.path = '' self.set_attr() def set_attr(self): def rm_sep(str:str): return str.replace(',', '').replace('.', '').replace(' ', '') item = self.item if item.count('**')%2 == 0: item = item.replace('**', '') if item.startswith('#') and (rm_sep(item.split('#',1)[1].split(' ',1)[0]).isdigit()): self.rank = int(rm_sep(item.split('#',1)[1].split(' - ',1)[0])) item = item.split(' - ', 1)[1] if ' - http' in item: self.url = 'http' + item.split(' - http',1)[1] item = item.split(' - http',1)[0] if ' - ')[0] item = item.split(' - ' in item) and ((item.count(' => ') == 2) or ('💞 => ' not in item)): if (' img' in item) or (' gif' in item): self.info = item.split(' => ')[-1] item = item[0:-(len(self.info)+4)] else: self.owner = item.split(' => ',1)[1] item = item.split(' => ',1)[0] if '💞 => ' in item: self.owner = item.split('💞 => ',1)[1] item = item.split('💞 => ',1)[0] if ':revolving_hearts: => ' in item: self.owner = item.split(':revolving_hearts: => ',1)[1] item = item.split(':revolving_hearts: => ',1)[0] if ' => ' in item: self.owner = item.split(' => ',1)[1] item = item.split(' => ',1)[0] self.name = item.strip() def __repr__(self): out_str = '' key_list = self.__dict__.keys() banned = ('item') keys = [key for key in key_list if (self.__dict__[key] and (key not in banned))] if keys: max_key_len = max([len(key[:6]) for key in keys]) else: return out_str for key in keys: if out_str: out_str+='\n' val = self.__dict__[key] key_val = len(val) if isinstance(val, list) else val out_str += f"├ {key[:6].replace('_',' ')}: {' '*(max_key_len-len(key))}{key_val}" return out_str def __bool__(self): return True if f'{self}' else False class character(): def __init__(self, attr:char_attr=char_attr()): if not isinstance(attr, char_attr): return print("attr is not char_attr type") self.attr = attr self.src = attr.path if attr.path else attr.url self.name = attr.name self.kakera = attr.kakera self.keys = attr.keys self.soul_keys = attr.soul_keys self.note = attr.note self.rank = attr.rank self.type = attr.type self.owner = attr.owner self.spheres = attr.spheres self.info = attr.info self.tags = attr.tags self.img_cnt = 0 self.off_cnt = 0 self.series = '' self.og_name = '' self.code = '' self.NSFW = False self.images:list[MudaeImage] = [] self.image = None self.mudae_image = None self.original_index = None self.is_playing = True self.src_fetch() if not len(self.images): return self.verify_images() self.set_mudae_image() def src_fetch(self): if self.src.startswith("https://mudae.net/uploads/"): code = self.src.split("https://mudae.net/uploads/",1)[1].split('/',1)[0] self.char_fetch(code) elif self.name: self.char_fetch(self.name) if self.src: im = MudaeImage(f'{len(self.images)+1}', "Custom", self.src) if im.image(): self.images.append(im) elif self.src: im = MudaeImage("1", "External", self.src) if im.image(): self.images.append(im) def verify_images(self): if (' img' not in self.info) and (' gif' not in self.info): return if not self.off_cnt: return if self.NSFW: return if not (self.code or self.name): return img_cnt_split = self.info.split(' img',1)[0].split(' ') if len(img_cnt_split) > 1: img_cnt_str = img_cnt_split[-1] else: img_cnt_str = img_cnt_split[0] if img_cnt_str.isdigit(): self.img_cnt += int(img_cnt_str) gif_cnt_split = self.info.split(' gif',1)[0].split(' ') if len(gif_cnt_split) > 1: gif_cnt_str = gif_cnt_split[-1] else: gif_cnt_str = gif_cnt_split[0] if gif_cnt_str.isdigit(): self.img_cnt += int(gif_cnt_str) img_diff = abs(self.off_cnt - self.img_cnt) if not img_diff: return print(f"removing {img_diff} image{' ' if img_diff==1 else 's'}") code = self.code if self.code else self.name chont_page = scraper.get(f"https://mudae.net/character/{code}/contributions") chont_soup = BeautifulSoup(chont_page.text, "html.parser") img_cont = chont_soup.find("div",class_="category",attrs={"data-category": "Character - Image"}) cont_container = img_cont.find("div", class_="contributions") contributions = cont_container.find_all("p", class_="contribution") for contrib in contributions: if not img_diff: return text = contrib.get_text(strip=False) if not "added image " in text: continue img_id = text.split("added image ")[1].split(' ',1)[0].strip() images = [] for img in self.images: if img_id not in img.src: if str(img.number+1) in img.title: img.title = img.title.replace(str(img.number+1), str(len(images)+1)) img.number = len(images) images.append(img) else: img_diff -= 1 self.images = images def char_fetch(self, code:str): if not code: return ### --- RATE LIMIT --- ### global tic delta = time.perf_counter() - tic time.sleep(max(0, SCRAPER_TIMEOUT_MS/1000-delta)) tic = time.perf_counter() ### --- RATE LIMIT --- ### char_page = scraper.get(f"https://mudae.net/character/{code}") ### --- BACK OFF --- ### if char_page.status_code != 200: print(f"'{code}' - no 200 code on url get, sleeping for 5s and trying again") time.sleep(5) char_page = scraper.get(f"https://mudae.net/character/{code}") if char_page.status_code != 200: print(f"'{code}' - no 200 code on url get again, removing character entry") return ### --- BACK OFF --- ### char_soup = BeautifulSoup(char_page.text, "html.parser") canonical = char_soup.find("link", rel="canonical") if canonical: canon_url = canonical.get("href") self.code = canon_url.split("https://mudae.net/character/",1)[1].split('/')[0] title = char_soup.find("meta", property="og:title") if title: title = title.get("content") if not title: title = char_soup.title.string title_split = title.split(" | ", 1) if len(title_split) == 2: self.og_name, rest = title_split if not self.name: self.name = self.og_name self.attr.name = self.name self.series = rest.replace(" - Mudae", "").strip() if char_soup.find("section", id="adult-warning"): self.NSFW = True else: if not self.type: gender_str = char_soup.select_one("#data-gender span.data").get_text(strip=True) if gender_str == 'Female': self.type = '($w)'; self.attr.type = self.type elif gender_str == 'Male': self.type = '($h)'; self.attr.type = self.type elif gender_str == 'Other': self.type = '($wh)'; self.attr.type = self.type else: self.type = '($wh)'; self.attr.type = self.type if not self.rank: rank_str = char_soup.select_one("#data-rank a.data").get_text(strip=True).lstrip("#").split('arrow',1)[0] if rank_str.isdigit(): self.rank = int(rank_str); self.attr.rank = self.rank container = char_soup.find("section", id="images") if not container: og_img = char_soup.find("meta", property="og:image") if og_img: og_img = og_img.get("content") if og_img: im = MudaeImage("1", "og_image", og_img) if im.image(): self.images.append(im) if (og_img != self.src) and self.src: im = MudaeImage(f'{len(self.images)+1}', "Imported", self.src) if im.image(): self.images.append(im) else: self.series = 'Custom' return for img in container.find_all("img"): try: self.images.append(MudaeImage(img.get("data-number"),img.get("alt"),img["src"])) except: continue self.off_cnt = len(self.images) def get_img_number_str(self): number_string = "" if len(self.images) > 1: number_string += f"{(self.mudae_image.number+1):,}/{len(self.images):,}" if self.NSFW: number_string += f"NSFW" return number_string def set_mudae_image(self, number=None): if (not self.src) and (number == None) and self.name: self.mudae_image = self.images[0] image = self.mudae_image.image() if image: self.image = image if self.original_index == None: self.original_index = self.mudae_image.number for img in self.images: if (number != img.number) and (number != None): continue if (self.src != img.src) and (number == None): continue image = img.image() if image: self.image = image self.mudae_image = img if self.original_index == None: self.original_index = self.mudae_image.number break def ec_command(self): if self.name: return f"$ec {self.name} $ {self.mudae_image.color}" else: return f"No character name, hex code: {self.mudae_image.color}" def c_command(self): if self.name: return f"$c {self.name} $ {self.mudae_image.number+1}" else: return f"No character name, image number: {self.mudae_image.number+1}" def next_image(self): if not self.mudae_image: return self.mudae_image.frame = 0 self.set_mudae_image((self.mudae_image.number+1) % (len(self.images))) self.mudae_image.frame = 0 def prev_image(self): if not self.mudae_image: return self.mudae_image.frame = 0 self.set_mudae_image((self.mudae_image.number-1) % (len(self.images))) self.mudae_image.frame = 0 def close(self): for img in self.images: img.close() self.images.clear() def __repr__(self): out_string = "\n" out_string += f"───────────────────╌┄┈\n" name = self.og_name if self.og_name else self.name if self.name: out_string += f" Name: {name}\n" if self.series: out_string += f" Series: {self.series}\n" if self.images: out_string += f" images: {len(self.images)}\n" if self.NSFW: out_string += f" NSFW\n" out_string += f"╭──────────────────╌┄┈\n" if self.images: out_string += f"├───────────────╮\n" out_string += f"╯ Current image │\n" out_string += f"╭───────────────╯\n" out_string += f"{self.mudae_image}\n" if self.attr: out_string += f"├────────────╮\n" out_string += f"╯ Attributes │\n" out_string += f"╭────────────╯\n" out_string += f"{self.attr}\n" out_string += f"╯" return out_string def __bool__(self): return bool(self.image) and bool(self.images) and bool(self.mudae_image)