reduce docker build context and match working baseline
This commit is contained in:
11
.dockerignore
Normal file
11
.dockerignore
Normal file
@@ -0,0 +1,11 @@
|
|||||||
|
.git
|
||||||
|
.gitignore
|
||||||
|
.dockerignore
|
||||||
|
worklogs
|
||||||
|
**/__pycache__
|
||||||
|
**/*.pyc
|
||||||
|
**/*.pyo
|
||||||
|
**/*.log
|
||||||
|
**/*.pid
|
||||||
|
nohup.out
|
||||||
|
qwen3_6_scripts/nohup.out
|
||||||
@@ -50,3 +50,4 @@ env:
|
|||||||
value: '1'
|
value: '1'
|
||||||
- name: LOOP1_NATIVE
|
- name: LOOP1_NATIVE
|
||||||
value: '1'
|
value: '1'
|
||||||
|
|
||||||
|
|||||||
@@ -335,10 +335,10 @@ def _get_seq_len_block_table_args(
|
|||||||
on the type of attention operation.
|
on the type of attention operation.
|
||||||
|
|
||||||
Decoder attn -> select entirely decoder self-attention-related fields
|
Decoder attn -> select entirely decoder self-attention-related fields
|
||||||
Encoder/decoder cross-attn -> select encoder sequence lengths &
|
Encoder/decoder cross-attn -> select encoder sequence lengths &
|
||||||
cross-attn block-tables fields
|
cross-attn block-tables fields
|
||||||
Encoder attn -> select encoder sequence lengths fields & no block tables
|
Encoder attn -> select encoder sequence lengths fields & no block tables
|
||||||
|
|
||||||
Arguments:
|
Arguments:
|
||||||
|
|
||||||
* attn_metadata: Attention metadata structure associated with attention op
|
* attn_metadata: Attention metadata structure associated with attention op
|
||||||
@@ -384,11 +384,11 @@ class XFormersMetadataBuilder(CommonMetadataBuilder[XFormersMetadata]):
|
|||||||
class XFormersImpl(AttentionImpl[XFormersMetadata]):
|
class XFormersImpl(AttentionImpl[XFormersMetadata]):
|
||||||
"""
|
"""
|
||||||
If the input tensors contain prompt tokens, the layout is as follows:
|
If the input tensors contain prompt tokens, the layout is as follows:
|
||||||
|<--------------- num_prefill_tokens ----------------->|
|
|<--------------- num_prefill_tokens ----------------->|
|
||||||
|<--prefill_0-->|<--prefill_1-->|...|<--prefill_N-1--->|
|
|<--prefill_0-->|<--prefill_1-->|...|<--prefill_N-1--->|
|
||||||
|
|
||||||
Otherwise, the layout is as follows:
|
Otherwise, the layout is as follows:
|
||||||
|<----------------- num_decode_tokens ------------------>|
|
|<----------------- num_decode_tokens ------------------>|
|
||||||
|<--decode_0-->|..........|<--decode_M-1-->|<--padding-->|
|
|<--decode_0-->|..........|<--decode_M-1-->|<--padding-->|
|
||||||
|
|
||||||
Generation tokens can contain padding when cuda-graph is used.
|
Generation tokens can contain padding when cuda-graph is used.
|
||||||
@@ -446,7 +446,7 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
|
|||||||
self.head_mapping = torch.repeat_interleave(
|
self.head_mapping = torch.repeat_interleave(
|
||||||
torch.arange(self.num_kv_heads, dtype=torch.int32),
|
torch.arange(self.num_kv_heads, dtype=torch.int32),
|
||||||
self.num_queries_per_kv)
|
self.num_queries_per_kv)
|
||||||
|
|
||||||
def forward(
|
def forward(
|
||||||
self,
|
self,
|
||||||
query: torch.Tensor,
|
query: torch.Tensor,
|
||||||
@@ -474,10 +474,10 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
|
|||||||
(1) key and value tensors were cached during prefill, and
|
(1) key and value tensors were cached during prefill, and
|
||||||
(2) cross-attention key and value tensors do not grow during
|
(2) cross-attention key and value tensors do not grow during
|
||||||
decode
|
decode
|
||||||
|
|
||||||
A note on how the attn_type (attention type enum) argument impacts
|
A note on how the attn_type (attention type enum) argument impacts
|
||||||
attention forward() behavior:
|
attention forward() behavior:
|
||||||
|
|
||||||
* DECODER: normal decoder-only behavior;
|
* DECODER: normal decoder-only behavior;
|
||||||
use decoder self-attention block table
|
use decoder self-attention block table
|
||||||
* ENCODER: no KV caching; pass encoder sequence
|
* ENCODER: no KV caching; pass encoder sequence
|
||||||
@@ -490,7 +490,7 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
|
|||||||
will match encoder sequence lengths, pass encoder sequence
|
will match encoder sequence lengths, pass encoder sequence
|
||||||
attributes to kernel (encoder_seq_lens/encoder_seq_lens_tensor/
|
attributes to kernel (encoder_seq_lens/encoder_seq_lens_tensor/
|
||||||
max_encoder_seq_len)
|
max_encoder_seq_len)
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
query: shape = [num_tokens, num_heads * head_size]
|
query: shape = [num_tokens, num_heads * head_size]
|
||||||
key: shape = [num_tokens, num_kv_heads * head_size]
|
key: shape = [num_tokens, num_kv_heads * head_size]
|
||||||
|
|||||||
Reference in New Issue
Block a user