From f7c031c0970736a9ab8302d903c582a8dfe72d5e Mon Sep 17 00:00:00 2001 From: unclecode Date: Thu, 9 May 2024 19:26:37 +0800 Subject: [PATCH] `chore: Remove unused code from test.py` --- test.py | 31 ------------------------------- 1 file changed, 31 deletions(-) delete mode 100644 test.py diff --git a/test.py b/test.py deleted file mode 100644 index 9fc26a30..00000000 --- a/test.py +++ /dev/null @@ -1,31 +0,0 @@ -from crawler.web_crawler import WebCrawler -from crawler.models import UrlModel -from crawler.utils import get_content_of_website -import os - -def main(): - # Initialize the WebCrawler with just the database path - crawler = WebCrawler(db_path='crawler_data.db') - - # Fetch a single page - single_url = UrlModel(url='https://kidocode.com', forced=True) - result = crawler.fetch_page( - single_url, - provider= "openai/gpt-3.5-turbo", - api_token = os.getenv('OPENAI_API_KEY'), - extract_blocks_flag=True, - word_count_threshold=5 - ) - print(result.model_dump()) - - # Fetch multiple pages - # urls = [ - # UrlModel(url='http://example.com', forced=False), - # UrlModel(url='http://example.org', forced=False) - # ] - # results = crawler.fetch_pages(urls, provider= "openai/gpt-4-turbo", api_token = os.getenv('OPENAI_API_KEY')) - # for res in results: - # print(res.model_copy()) - -if __name__ == '__main__': - main() \ No newline at end of file