add use proxy and llm baseurl examples
This commit is contained in:
27
README.md
27
README.md
@@ -190,6 +190,33 @@ result = crawler.run(
|
|||||||
print(result.extracted_content)
|
print(result.extracted_content)
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### Extract Structured Data from Web Pages With Proxy and BaseUrl
|
||||||
|
|
||||||
|
```python
|
||||||
|
from crawl4ai import WebCrawler
|
||||||
|
from crawl4ai.extraction_strategy import LLMExtractionStrategy
|
||||||
|
|
||||||
|
def create_crawler():
|
||||||
|
crawler = WebCrawler(verbose=True, proxy="http://127.0.0.1:7890")
|
||||||
|
crawler.warmup()
|
||||||
|
return crawler
|
||||||
|
|
||||||
|
crawler = create_crawler()
|
||||||
|
|
||||||
|
crawler.warmup()
|
||||||
|
|
||||||
|
result = crawler.run(
|
||||||
|
url="https://www.nbcnews.com/business",
|
||||||
|
extraction_strategy=LLMExtractionStrategy(
|
||||||
|
provider="openai/gpt-4o",
|
||||||
|
api_token="sk-",
|
||||||
|
base_url="https://api.openai.com/v1"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
print(result.markdown)
|
||||||
|
```
|
||||||
|
|
||||||
## Documentation 📚
|
## Documentation 📚
|
||||||
|
|
||||||
For detailed documentation, including installation instructions, advanced features, and API reference, visit our [Documentation Website](https://crawl4ai.com/mkdocs/).
|
For detailed documentation, including installation instructions, advanced features, and API reference, visit our [Documentation Website](https://crawl4ai.com/mkdocs/).
|
||||||
|
|||||||
Reference in New Issue
Block a user