apiVersion: apps/v1
kind: Deployment
metadata:
  name: litellm-proxy
  namespace: your-namespace
  labels:
    app: litellm-proxy
spec:
  replicas: 1
  selector:
    matchLabels:
      app: litellm-proxy
  template:
    metadata:
      labels:
        app: litellm-proxy
    spec:
      nodeName: <your-node-name>
      containers:
        - name: litellm-container
          image: litellm/litellm:latest
          imagePullPolicy: IfNotPresent #Always

          env:
            - name: LITELLM_REQUEST_TIMEOUT
              value: "120"
            - name: LITELLM_STREAM_TIMEOUT
              value: "120"
            - name: LITELLM_MODEL_TIMEOUT
              value: "180"

          command: ["litellm"]
          args:
            - "--config"
            - "/app/litellm-config.yaml"
            - "--port"
            - "4000"

          ports:
            - name: http
              containerPort: 4000
              protocol: TCP

          volumeMounts:
            - name: config-volume
              mountPath: /app/litellm-config.yaml
              subPath: litellm-config.yaml

          livenessProbe:
            tcpSocket:
              port: 4000
            initialDelaySeconds: 120
            periodSeconds: 15
            failureThreshold: 3
          readinessProbe:
            httpGet:
              path: /health/readiness
              port: 4000
            initialDelaySeconds: 120
            periodSeconds: 15
            failureThreshold: 3

      volumes:
        - name: config-volume
          configMap:
            name: litellm-config

      restartPolicy: Always
      terminationGracePeriodSeconds: 30

  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxUnavailable: 0
      maxSurge: 1
---
apiVersion: v1
kind: Service
metadata:
  name: litellm-proxy
  namespace: your-namespace
  annotations:
    service.beta.kubernetes.io/alibaba-cloud-loadbalancer-address-type: intranet
    service.beta.kubernetes.io/alibaba-cloud-loadbalancer-force-override-listeners: "true"
    service.beta.kubernetes.io/alibaba-cloud-loadbalancer-ip: "<your-internal-lb-ip>"
spec:
  type: LoadBalancer
  externalTrafficPolicy: "Local"
  selector:
    app: litellm-proxy
  ports:
    - name: http
      port: 4000
      protocol: TCP
      targetPort: 4000