Snapshot of main at 9f39a6e1e (2026-11-02)
This commit is contained in:
Ronald Huynen committed
commit
6ccf1792d7
2580 files changed
+1040202
No files matched your search
Executable
+208
@@ -0,0 +1,208 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Colors for output
|
||||
GREEN='\033[0;32m'
|
||||
BLUE='\033[0;34m'
|
||||
YELLOW='\033[0;33m'
|
||||
RED='\033[0;31m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Function for printing section headers
|
||||
section_header() {
|
||||
printf "\n${BLUE}===========================================================${NC}\n"
|
||||
printf "${BLUE} $1${NC}\n"
|
||||
printf "${BLUE}===========================================================${NC}\n\n"
|
||||
}
|
||||
|
||||
# Error handling
|
||||
error_exit() {
|
||||
printf "${RED}ERROR: $1${NC}\n" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Load .env variables
|
||||
set -a
|
||||
if [ -f .env ]; then
|
||||
. ./.env 2>/dev/null
|
||||
fi
|
||||
set +a
|
||||
|
||||
# Set Elasticsearch authentication flag if credentials exist
|
||||
if [ -n "$ELASTICSEARCH_USER" ] && [ -n "$ELASTICSEARCH_PASSWORD" ]; then
|
||||
ES_AUTH="-u $ELASTICSEARCH_USER:$ELASTICSEARCH_PASSWORD"
|
||||
else
|
||||
ES_AUTH=""
|
||||
fi
|
||||
|
||||
section_header "🗑️ CLEANING UP OLD INDICES AND ALIASES"
|
||||
|
||||
# Function to delete all indices matching a pattern
|
||||
delete_indices_by_pattern() {
|
||||
local pattern=$1
|
||||
local alias_name=$2
|
||||
|
||||
printf "${YELLOW}Cleaning up pattern: ${pattern}${NC}\n"
|
||||
|
||||
# Get all indices matching the pattern (both direct indices and versioned)
|
||||
local all_indices=$(curl $ES_AUTH -s -X GET "localhost:9200/_cat/indices?h=index" | grep -E "^${pattern}(_[0-9]+)?$" | sort)
|
||||
printf "${YELLOW}Found indices for pattern '${pattern}': ${all_indices}${NC}\n"
|
||||
|
||||
# Check if there's a direct index with the alias name (this causes conflicts)
|
||||
local direct_index=$(echo "$all_indices" | grep -E "^${pattern}$")
|
||||
if [ -n "$direct_index" ]; then
|
||||
printf "${YELLOW}Found conflicting direct index: $direct_index (this will be deleted)${NC}\n"
|
||||
delete_response=$(curl $ES_AUTH -s -X DELETE "localhost:9200/$direct_index")
|
||||
printf "${YELLOW}Delete response: $delete_response${NC}\n"
|
||||
if echo "$delete_response" | grep -q '"acknowledged":true'; then
|
||||
printf "${GREEN}Successfully deleted conflicting index: $direct_index${NC}\n"
|
||||
else
|
||||
printf "${RED}Failed to delete conflicting index $direct_index — attempting force delete${NC}\n"
|
||||
curl $ES_AUTH -s -X DELETE "localhost:9200/$direct_index?ignore_unavailable=true"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Delete ALL versioned indices — scout:import will create a fresh one
|
||||
local versioned_indices=$(echo "$all_indices" | grep -E "^${pattern}_[0-9]+$")
|
||||
|
||||
if [ -n "$versioned_indices" ]; then
|
||||
for index in $versioned_indices; do
|
||||
printf "${YELLOW}Deleting versioned index: $index${NC}\n"
|
||||
curl $ES_AUTH -s -X DELETE "localhost:9200/$index?ignore_unavailable=true" || printf "${RED}Failed to delete $index${NC}\n"
|
||||
done
|
||||
fi
|
||||
|
||||
# Remove any existing alias (safe operation)
|
||||
printf "${YELLOW}Removing existing alias: $alias_name${NC}\n"
|
||||
curl $ES_AUTH -s -X POST "localhost:9200/_aliases" -H 'Content-Type: application/json' -d"{\"actions\":[{\"remove\":{\"alias\":\"$alias_name\",\"index\":\"*\"}}]}" 2>/dev/null || printf "${YELLOW}Alias $alias_name didn't exist${NC}\n"
|
||||
}
|
||||
|
||||
# Define index patterns (using regular variables instead of associative array)
|
||||
INDEX_PATTERNS="users_index organizations_index banks_index posts_index calls_index transactions_index"
|
||||
|
||||
# Clean up old indices for each pattern
|
||||
for alias in $INDEX_PATTERNS; do
|
||||
delete_indices_by_pattern "$alias" "$alias"
|
||||
done
|
||||
|
||||
section_header "⏳ WAITING FOR ELASTICSEARCH CLUSTER HEALTH"
|
||||
|
||||
# Wait for Elasticsearch to be ready — require green (all shards assigned) before importing.
|
||||
# On production single-node clusters this can take up to 2 minutes after a restart.
|
||||
printf "${YELLOW}Waiting for green cluster health (up to 120s)...${NC}\n"
|
||||
timeout=120
|
||||
counter=0
|
||||
cluster_ready=false
|
||||
while [ $counter -lt $timeout ]; do
|
||||
health_response=$(curl $ES_AUTH -s "localhost:9200/_cluster/health" 2>/dev/null)
|
||||
if echo "$health_response" | grep -q '"status":"green"'; then
|
||||
printf "${GREEN}Cluster is green — all shards active!${NC}\n"
|
||||
cluster_ready=true
|
||||
break
|
||||
fi
|
||||
# Also accept yellow on single-node clusters where replicas can never be assigned
|
||||
unassigned=$(echo "$health_response" | grep -o '"unassigned_shards":[0-9]*' | grep -o '[0-9]*$')
|
||||
initializing=$(echo "$health_response" | grep -o '"initializing_shards":[0-9]*' | grep -o '[0-9]*$')
|
||||
if echo "$health_response" | grep -q '"status":"yellow"' && [ "${unassigned:-1}" = "0" ] && [ "${initializing:-1}" = "0" ]; then
|
||||
printf "${GREEN}Cluster is yellow with 0 unassigned/initializing shards — safe to proceed.${NC}\n"
|
||||
cluster_ready=true
|
||||
break
|
||||
fi
|
||||
printf "${YELLOW}Waiting for cluster (${counter}/${timeout}) — status: $(echo "$health_response" | grep -o '"status":"[^"]*"'), unassigned: ${unassigned:-?}...${NC}\n"
|
||||
sleep 3
|
||||
counter=$((counter + 3))
|
||||
done
|
||||
|
||||
if [ "$cluster_ready" = false ]; then
|
||||
printf "${RED}ERROR: Elasticsearch cluster did not reach a healthy state after ${timeout}s.${NC}\n"
|
||||
printf "${RED}Check: curl -s localhost:9200/_cluster/health?pretty${NC}\n"
|
||||
printf "${RED}Check: curl -s localhost:9200/_cat/shards?v | grep -v STARTED${NC}\n"
|
||||
error_exit "Elasticsearch cluster not healthy — aborting re-index to prevent partial data"
|
||||
fi
|
||||
|
||||
# Helper: import a model with up to 3 retries (15s pause between attempts)
|
||||
import_with_retry() {
|
||||
local model="$1"
|
||||
local attempts=3
|
||||
local attempt=1
|
||||
while [ $attempt -le $attempts ]; do
|
||||
printf "${GREEN}Importing %s (attempt %d/%d)...${NC}\n" "$model" "$attempt" "$attempts"
|
||||
if SCOUT_QUEUE=false php artisan scout:import "$model"; then
|
||||
printf "${GREEN}Successfully imported %s${NC}\n" "$model"
|
||||
return 0
|
||||
fi
|
||||
printf "${YELLOW}Import of %s failed on attempt %d.${NC}\n" "$model" "$attempt"
|
||||
if [ $attempt -lt $attempts ]; then
|
||||
printf "${YELLOW}Waiting 15s before retry...${NC}\n"
|
||||
sleep 15
|
||||
fi
|
||||
attempt=$((attempt + 1))
|
||||
done
|
||||
error_exit "Failed to import $model after $attempts attempts"
|
||||
}
|
||||
|
||||
section_header "🧹 SKIPPING FLUSH - DIRECT IMPORT WILL CREATE FRESH INDICES"
|
||||
|
||||
# Skip flush since we already cleaned up indices above and import will create fresh ones
|
||||
printf "${YELLOW}Note: Skipping scout:flush commands to avoid shard conflicts.${NC}\n"
|
||||
printf "${YELLOW}The scout:import commands will create fresh indices automatically.${NC}\n"
|
||||
|
||||
section_header "📥 IMPORTING ALL MODELS TO ELASTICSEARCH (WITHOUT QUEUE)"
|
||||
|
||||
import_with_retry "App\Models\User"
|
||||
sleep 5
|
||||
|
||||
import_with_retry "App\Models\Organization"
|
||||
sleep 5
|
||||
|
||||
import_with_retry "App\Models\Bank"
|
||||
sleep 5
|
||||
|
||||
import_with_retry "App\Models\Post"
|
||||
sleep 5
|
||||
|
||||
import_with_retry "App\Models\Call"
|
||||
sleep 5
|
||||
|
||||
import_with_retry "App\Models\Transaction"
|
||||
|
||||
# Wait a moment for Elasticsearch to process (shorter wait since we're not using queue)
|
||||
printf "${YELLOW}Waiting for Elasticsearch to process...${NC}\n"
|
||||
sleep 2
|
||||
|
||||
section_header "🔗 CREATING ALIASES FOR STABLE INDEX NAMES"
|
||||
|
||||
# Create aliases pointing to the new timestamped indices
|
||||
for alias in $INDEX_PATTERNS; do
|
||||
printf "${GREEN}Creating alias for: $alias${NC}\n"
|
||||
|
||||
# Find the latest index for this pattern
|
||||
latest_index=$(curl $ES_AUTH -s -X GET "localhost:9200/_cat/indices?h=index" | grep "^${alias}_" | sort | tail -n 1)
|
||||
|
||||
if [ -n "$latest_index" ]; then
|
||||
printf "${GREEN}Found latest index: $latest_index${NC}\n"
|
||||
|
||||
# Create alias
|
||||
curl $ES_AUTH -s -X POST "localhost:9200/_aliases" -H 'Content-Type: application/json' -d"
|
||||
{
|
||||
\"actions\": [
|
||||
{ \"add\": { \"index\": \"${latest_index}\", \"alias\": \"${alias}\" } }
|
||||
]
|
||||
}
|
||||
" && printf "${GREEN}✅ Created alias ${alias} -> ${latest_index}${NC}\n" || printf "${RED}❌ Failed to create alias ${alias}${NC}\n"
|
||||
else
|
||||
printf "${RED}❌ No index found for pattern ${alias}_*${NC}\n"
|
||||
fi
|
||||
done
|
||||
|
||||
section_header "📋 FINAL ELASTICSEARCH INDICES AND ALIASES"
|
||||
|
||||
printf "${BLUE}Indices:${NC}\n"
|
||||
curl $ES_AUTH -s -X GET "localhost:9200/_cat/indices?v"
|
||||
|
||||
printf "\n${BLUE}Aliases:${NC}\n"
|
||||
curl $ES_AUTH -s -X GET "localhost:9200/_cat/aliases?v"
|
||||
|
||||
section_header "✅ REINDEXING COMPLETE"
|
||||
|
||||
printf "${GREEN}All models have been reindexed successfully!${NC}\n"
|
||||
printf "${GREEN}You can now use the stable alias names (users_index, organizations_index, etc.) in your application.${NC}\n"
|
||||
Reference in new issue
Block a user