Repository navigation
Expand file tree
/
Copy pathaicoder.service
More file actions
28 lines (25 loc) · 2.69 KB
/
Copy pathaicoder.service
File metadata and controls
28 lines (25 loc) · 2.69 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
[Unit]
Description=CoderDeepSeek
After=network-online.target
Wants=network-online.target
[Service]
Type=simple
User=root
#Group=neo
WorkingDirectory=/ai/llama.cpp
RestartSec=160s
Restart=always
ExecStart=/ai/llama.cpp/build/bin/llama-server -m /ai/models/Devstral-Small-2507-Q6_K.gguf -ngl 130 -np 1 --threads -1 --top-k 64 --min-p 0.01 --top-p 0.95 -fa --device CUDA0,CUDA1 --host 192.168.0.55 --port 5000 --ctx-size 131072 --temp 0.15 --prio 2 --repeat-penalty 1.0 -ts 0.5,0.5 -sm row --no-mmap --numa distribute --reasoning-format none --cache-type-k q8_0
#/ai/llama.cpp/build/bin/llama-server -m /ai/models/Qwen3-32B-Q8_0.gguf -ngl 130 -np 1 --top-k 20 --min-p 0.01 --top-p 0.8 -fa --host 192.168.0.55 --port 5000 --model-draft /ai/models/Qwen3-0.6B-Q8_0.gguf -ngld 99 --draft-max 16 --draft-min 4 --draft-p-min 0.01 --ctx-size 32768 -cd 8192 -ctk q8_0 -ctv q8_0 -ts 0.5,0.5 -sm row -e --jinja --chat-template-file /ai/models/templates/qwen3.jinja --threads 32 --threads-batch 32 --rope-scaling linear
#ExecStart=/ai/llama.cpp/build/bin/llama-server -m /ai/models/qwen2.5-coder-32b-instruct-q6_k.gguf -ngl 130 --host 192.168.0.55 --port 5000 -ts 0.5,0.5 -sm row -fa -np 1 --temp 0.6 --top-k 20 --min-p 0.0 --top-p 0.95 -c 32768 --model-draft /ai/models/qwen2.5-coder-0.5b-instruct-q8_0.gguf -ngld 99 --draft-max 16 --draft-min 4 --draft-p-min 0.4 --device CUDA0,CUDA1 --device-draft CUDA1,CUDA0 -np 1 --top-k 64 --min-p 0.01 --top-p 0.95 -fa --ctx-size 32768 --repeat-penalty 1.1 --no-mmap --reasoning-format none
#--cache-type-k q8_0
#/ai/llama.cpp/build/bin/llama-server -m /ai/models/qwen2.5-coder-32b-instruct-q6_k.gguf -ngl 130 --host 192.168.0.55 --port 5000 -ts 0.5,0.5 -sm row -fa -np 1 --temp 0.6 --top-k 20 --min-p 0.0 --top-p 0.95 -c 32768 --model-draft /ai/models/qwen2.5-coder-0.5b-instruct-q8_0.gguf -ngld 99 --draft-max 16 --draft-min 4 --draft-p-min 0.4 --device CUDA0,CUDA1 --device-draft CUDA1,CUDA0
#ExecStart=/ai/llama.cpp/build/bin/llama-server -m /ai/models/qwen2.5-coder-32b-instruct-q8_0.gguf -ngl 130 --host 192.168.0.55 --port 5000 -ts 0.5,0.5 -sm row -fa -np 1 --temp 0.7 --top-k 20 --min-p 0.0 --top-p 0.8 -ctv q8_0 -ctk q8_0 -c 32000
#/ai/llama.cpp/build/bin/llama-server -m /ai/models/Qwen2.5-Coder-32B-Instruct-Q8_0.gguf -ngl 130 --host 192.168.0.55 --port 5000 -ts 0.5,0.5 -sm row -fa -np 1 --temp 0.01 --top-k 0 --min-p 0.0 --top-p 0.95 -ctv q8_0 -ctk q8_0 -c 32000
StandardOutput=inherit
StandardError=inherit
#KillMode=process
Restart=on-failure
Environment="CUDA_VISIBLE_DEVICES=0,1 LLAMA_CUDA_FORCE_MMQ=ON LLAMA_CUDA_F16=0 MATMUL_INT8=1 GPU_CHOICE=A USE_CUDA118=TRUE LAUNCH_AFTER_INSTALL=TRUE INSTALL_EXTENSIONS=FALSE LLAMA_CUBLAS=on LLAMA_CUDA_FORCE_MMQ=ON"
[Install]
WantedBy=multi-user.target