export_log_k8s.sh 2.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869
  1. #!/bin/bash
  2. # Exit immediately for non zero status
  3. set -e
  4. ns_name=$1
  5. instance_name=$2
  6. log_dir=${3:-"k8s_logs"}
  7. #show proxy pod log
  8. array=($(kubectl get pod -n ${ns_name} -l "component=proxy, app.kubernetes.io/instance=${instance_name}"| awk 'NR == 1 {next} {print $1}'))
  9. echo ${array[@]}
  10. for pod in ${array[*]}
  11. do
  12. echo "show log of proxy pod $pod "
  13. kubectl logs $pod -n ${ns_name} --tail=100 || echo "show log for pod $pod failed"
  14. done
  15. # export info of etcd
  16. array=($(kubectl get pod -n ${ns_name} -l "app.kubernetes.io/name=etcd, app.kubernetes.io/instance=${instance_name}"| awk 'NR == 1 {next} {print $1}'))
  17. echo ${array[@]}
  18. mkdir -p $log_dir/etcd_session
  19. for pod in ${array[*]}
  20. do
  21. echo "check session for etcd pod $pod "
  22. kubectl exec $pod -n ${ns_name} -- etcdctl get --prefix by-dev/meta/session > ./$log_dir/etcd_session/$pod.log || echo "export session for pod $pod failed"
  23. done
  24. echo "check session done"
  25. # export logs of all pods
  26. array_1=($(kubectl get pod -n ${ns_name} -l "app.kubernetes.io/instance=${instance_name}"| awk 'NR == 1 {next} {print $1}'))
  27. array_2=($(kubectl get pod -n ${ns_name} -l "release=${instance_name}"| awk 'NR == 1 {next} {print $1}'))
  28. array=(${array_1[@]} ${array_2[@]})
  29. echo ${array[@]}
  30. if [ ! -d $log_dir/pod_log ] || [ ! -d $log_dir/pod_describe ];
  31. then
  32. mkdir -p $log_dir/pod_log
  33. mkdir -p $log_dir/pod_log_previous
  34. mkdir -p $log_dir/pod_describe
  35. fi
  36. echo "export logs start"
  37. for pod in ${array[*]}
  38. do
  39. echo "export logs for pod $pod "
  40. kubectl logs $pod -n ${ns_name} > ./$log_dir/pod_log/$pod.log 2>&1 || echo "export log for pod $pod failed"
  41. kubectl logs $pod --previous -n ${ns_name} > ./$log_dir/pod_log_previous/$pod.log 2>&1 || echo "pod $pod has no previous log"
  42. kubectl describe pod $pod -n ${ns_name} > ./$log_dir/pod_describe/$pod.log 2>&1 || echo "describe pod $pod failed"
  43. done
  44. echo "export logs done"
  45. # export goroutine of all milvus pods
  46. array=($(kubectl get pod -n ${ns_name} -l "app.kubernetes.io/instance=${instance_name}, app.kubernetes.io/name=milvus"| awk 'NR == 1 {next} {print $1}'))
  47. echo ${array[@]}
  48. if [ ! -d $log_dir/goroutine ];
  49. then
  50. mkdir -p $log_dir/goroutine
  51. fi
  52. echo "export goroutine start"
  53. for pod in ${array[*]}
  54. do
  55. echo "export goroutine for pod $pod "
  56. ip=($(kubectl get pod ${pod} -n ${ns_name} -o jsonpath='{.status.podIP}'))
  57. echo $ip
  58. curl "http://${ip}:9091/debug/pprof/goroutine?debug=10" -o ./$log_dir/goroutine/$pod.log || echo "export goroutine for pod $pod failed"
  59. done
  60. echo "export goroutine done"