ayrisdev/crawl4ai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf(installer): use git sparse-checkout to download only cnode_pkg directory

Browse Source 
- Only fetches deploy/installer/cnode_pkg/ instead of entire repo
- Uses --depth=1 for minimal git history
- Faster download and smaller footprint
- Requires git (added check)
This commit is contained in:
unclecode
2025-10-21 09:40:22 +08:00
parent 9faddd30f5
commit 02f0e4787a
1 changed files with 26 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
deploy/installer/deploy.sh
View File

@@ -71,42 +71,40 @@ fi
TMP_DIR="$(mktemp -d)" TMP_DIR="$(mktemp -d)"
cd "$TMP_DIR" cd "$TMP_DIR"
# Download cnode package from GitHub # Download only cnode_pkg from GitHub using sparse checkout
echo -e "\n${BLUE}Downloading cnode package from GitHub...${NC}" echo -e "\n${BLUE}Downloading cnode package from GitHub...${NC}"
PACKAGE_URL="https://github.com/$GITHUB_REPO/archive/refs/heads/$BRANCH.tar.gz"
if command -v curl &> /dev/null; then if ! command -v git &> /dev/null; then
if ! curl -fsSL "$PACKAGE_URL" -o cnode.tar.gz; then echo -e "${RED}Error: git is required but not found${NC}"
echo -e "${RED}Error: Failed to download package${NC}" echo -e "${YELLOW}Install git and try again${NC}"
rm -rf "$TMP_DIR" rm -rf "$TMP_DIR"
exit 1 exit 1
fi fi
elif command -v wget &> /dev/null; then
if ! wget -q "$PACKAGE_URL" -O cnode.tar.gz; then # Initialize sparse checkout
echo -e "${RED}Error: Failed to download package${NC}" git init -q
rm -rf "$TMP_DIR" git remote add origin "https://github.com/$GITHUB_REPO.git"
exit 1 git config core.sparseCheckout true
fi
else # Only checkout the cnode_pkg directory
echo -e "${RED}Error: Neither curl nor wget found${NC}" echo "deploy/installer/cnode_pkg/*" > .git/info/sparse-checkout
# Pull only the needed files
if ! git pull -q --depth=1 origin "$BRANCH"; then
echo -e "${RED}Error: Failed to download package${NC}"
rm -rf "$TMP_DIR"
exit 1
fi
if [ ! -d "deploy/installer/cnode_pkg" ]; then
echo -e "${RED}Error: Package directory not found${NC}"
rm -rf "$TMP_DIR" rm -rf "$TMP_DIR"
exit 1 exit 1
fi fi
echo -e "${GREEN}✓ Package downloaded${NC}" echo -e "${GREEN}✓ Package downloaded${NC}"
# Extract package REPO_DIR="."
echo -e "${BLUE}Extracting package...${NC}"
tar -xzf cnode.tar.gz
REPO_DIR=$(find . -maxdepth 1 -type d -name "crawl4ai-*" | head -1)
if [ -z "$REPO_DIR" ] || [ ! -d "$REPO_DIR/deploy/installer/cnode_pkg" ]; then
echo -e "${RED}Error: Invalid package structure${NC}"
rm -rf "$TMP_DIR"
exit 1
fi
echo -e "${GREEN}✓ Package extracted${NC}"
# Install Python dependencies # Install Python dependencies
echo -e "\n${BLUE}Installing Python dependencies...${NC}" echo -e "\n${BLUE}Installing Python dependencies...${NC}"