Skip to content

Commit 29b52d3

Browse files
authored
model : Inkling 2 Mini (#32694)
* Conversion * Separate model_type * Add gate_rank to allow building from metadata while testing * Base model * Loop graph * Support the new is_varloop attribute * Adopt naming scheme from #32588 * Rename to varying
1 parent e3c3f66 commit 29b52d3

13 files changed

Lines changed: 631 additions & 123 deletions

‎conversion/__init__.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -148,7 +148,9 @@
148148
"HYV4ForCausalLM": "hy_v4",
149149
"IQuestCoderForCausalLM": "llama",
150150
"Inkling2ForCausalLM": "inkling2",
151+
"Inkling2LoopForCausalLM": "inkling2",
151152
"Inkling2ForConditionalGeneration": "inkling2",
153+
"Inkling2LoopForConditionalGeneration": "inkling2",
152154
"InternLM2ForCausalLM": "internlm",
153155
"InternLM3ForCausalLM": "internlm",
154156
"JAISLMHeadModel": "jais",
@@ -389,6 +391,7 @@
389391
"HunYuanVLForConditionalGeneration": "hunyuan",
390392
"Idefics3ForConditionalGeneration": "smolvlm",
391393
"Inkling2ForConditionalGeneration": "inkling2",
394+
"Inkling2LoopForConditionalGeneration": "inkling2",
392395
"InternVisionModel": "internvl",
393396
"JanusForConditionalGeneration": "januspro",
394397
"KimiK25ForConditionalGeneration": "kimivl",

‎conversion/inkling2.py‎

Lines changed: 57 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ def _is_inkling2(dir_model: Path) -> bool:
6969
if not (dir_model / "config.json").is_file():
7070
return False
7171
with open(dir_model / "config.json", encoding="utf-8") as f:
72-
return json.load(f).get("model_type") in ("inkling2", "inkling2_text")
72+
return json.load(f).get("model_type") in ("inkling2", "inkling2_text", "inkling2_loop", "inkling2_loop_text")
7373

7474

7575
@ModelBase.register_hparams_loader(_is_inkling2)
@@ -274,7 +274,62 @@ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iter
274274
yield from super().modify_tensors(data_torch, name, bid)
275275

276276

277-
@ModelBase.register("Inkling2ForConditionalGeneration")
277+
@ModelBase.register("Inkling2LoopForConditionalGeneration", "Inkling2LoopForCausalLM")
278+
class Inkling2LoopModel(Inkling2Model):
279+
model_arch = gguf.MODEL_ARCH.INKLING2_LOOP
280+
281+
# raw checkpoint and transformers names of the cross-pass message queries
282+
_query_kinds = {
283+
"model.middle_cycle_cross_pass_queries": "q",
284+
"model.middle_cycle_cross_pass_kv_queries": "kv",
285+
"model.cross_pass_messages.queries": "q",
286+
"model.cross_pass_messages.kv_queries": "kv",
287+
}
288+
289+
def __init__(self, *args, **kwargs):
290+
super().__init__(*args, **kwargs)
291+
self._queries: dict[str, Tensor] = {}
292+
self.loop = self.hparams["loop_config"]
293+
if set(self.loop) != {"core_start", "core_end", "num_passes", "gate_rank", "is_varloop"}:
294+
raise ValueError(f"Inkling2Loop: unsupported loop_config keys {sorted(self.loop)}")
295+
start, end, block = self.loop["core_start"], self.loop["core_end"], self.hparams["n_layers_per_attn_res_block"]
296+
# a fixed loop needs at least 2 passes, a variable loop at least 1
297+
min_passes = 1 if self.loop["is_varloop"] else 2
298+
if not 0 < start < end <= self.block_count or start % block or end % block or self.loop["num_passes"] < min_passes or self.loop["gate_rank"] < 1:
299+
raise ValueError(f"Inkling2Loop: invalid loop_config {self.loop}")
300+
301+
def set_gguf_parameters(self):
302+
super().set_gguf_parameters()
303+
self.gguf_writer.add_loop_start(self.loop["core_start"])
304+
self.gguf_writer.add_loop_end(self.loop["core_end"])
305+
self.gguf_writer.add_loop_count(self.loop["num_passes"])
306+
self.gguf_writer.add_loop_gate_rank(self.loop["gate_rank"])
307+
self.gguf_writer.add_loop_varying(self.loop["is_varloop"])
308+
309+
def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool:
310+
# the reference does not quantize the loop tensors
311+
if new_name.endswith(("loop_msg_query.weight", "_ctrl_down.weight", "_ctrl_gain.weight", "_ctrl_forget.weight")):
312+
return gguf.GGMLQuantizationType.F32
313+
return super().tensor_force_quant(name, new_name, bid, n_dims)
314+
315+
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
316+
if name not in self._query_kinds:
317+
yield from super().modify_tensors(data_torch, name, bid)
318+
return
319+
self._queries[self._query_kinds[name]] = data_torch
320+
if len(self._queries) < 2:
321+
return
322+
q, kv = self._queries["q"], self._queries["kv"]
323+
start = self.loop["core_start"]
324+
if q.shape[0] != 2 * (self.loop["core_end"] - start) or kv.shape != q.shape:
325+
raise ValueError(f"Inkling2Loop: cross-pass queries {tuple(q.shape)} and {tuple(kv.shape)} do not match the loop core")
326+
# core layer start + i mixes its q, k, v and mlp messages with these four queries
327+
for i in range(q.shape[0] // 2):
328+
query = torch.stack([q[2 * i], kv[2 * i], kv[2 * i + 1], q[2 * i + 1]])
329+
yield self.format_tensor_name(gguf.MODEL_TENSOR.LOOP_MSG_QUERY, start + i, ".weight"), query
330+
331+
332+
@ModelBase.register("Inkling2ForConditionalGeneration", "Inkling2LoopForConditionalGeneration")
278333
class Inkling2VisionModel(MmprojModel):
279334
def get_vision_config(self):
280335
config = self.global_config.get("vision_config")

‎gguf-py/gguf/constants.py‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -326,6 +326,13 @@ class HRM:
326326
L_CYCLES = "{arch}.hrm.l_cycles"
327327
PREFIX_LM = "{arch}.hrm.prefix_lm"
328328

329+
class Loop:
330+
START = "{arch}.loop.start"
331+
END = "{arch}.loop.end"
332+
COUNT = "{arch}.loop.count"
333+
GATE_RANK = "{arch}.loop.gate_rank"
334+
VARYING = "{arch}.loop.varying"
335+
329336
class Metis:
330337
PASS_COUNT = "{arch}.metis.pass_count"
331338
MEMORY_LENGTH = "{arch}.metis.memory_length"
@@ -778,6 +785,7 @@ class MODEL_ARCH(IntEnum):
778785
KOLIBRI1 = auto()
779786
LIMITE = auto()
780787
INKLING2 = auto()
788+
INKLING2_LOOP = auto()
781789
DOMYN_EDGE = auto()
782790

783791

@@ -880,6 +888,19 @@ class MODEL_TENSOR(IntEnum):
880888
MOE_LATENT_UP = auto() # nemotron 3 super
881889
FFN_POST_CONV = auto()
882890
FFN_RES_QUERY = auto()
891+
LOOP_MSG_QUERY = auto()
892+
LOOP_Q_CTRL_DOWN = auto()
893+
LOOP_Q_CTRL_GAIN = auto()
894+
LOOP_Q_CTRL_FORGET = auto()
895+
LOOP_K_CTRL_DOWN = auto()
896+
LOOP_K_CTRL_GAIN = auto()
897+
LOOP_K_CTRL_FORGET = auto()
898+
LOOP_V_CTRL_DOWN = auto()
899+
LOOP_V_CTRL_GAIN = auto()
900+
LOOP_V_CTRL_FORGET = auto()
901+
LOOP_FFN_CTRL_DOWN = auto()
902+
LOOP_FFN_CTRL_GAIN = auto()
903+
LOOP_FFN_CTRL_FORGET = auto()
883904
STREAM_EMBD = auto()
884905
DEPTH_MEM_STATE_WRITE = auto()
885906
DEPTH_MEM_META_WRITE = auto()
@@ -1779,6 +1800,7 @@ class MODEL_TENSOR(IntEnum):
17791800
MODEL_ARCH.KOLIBRI1: "kolibri1",
17801801
MODEL_ARCH.LIMITE: "limite",
17811802
MODEL_ARCH.INKLING2: "inkling2",
1803+
MODEL_ARCH.INKLING2_LOOP: "inkling2_loop",
17821804
MODEL_ARCH.DOMYN_EDGE: "domynedge",
17831805
}
17841806

@@ -1881,6 +1903,19 @@ class MODEL_TENSOR(IntEnum):
18811903
MODEL_TENSOR.MOE_LATENT_UP: "blk.{bid}.ffn_latent_up", # nemotron 3 super
18821904
MODEL_TENSOR.FFN_POST_CONV: "blk.{bid}.ffn_post_conv",
18831905
MODEL_TENSOR.FFN_RES_QUERY: "blk.{bid}.ffn_res_query",
1906+
MODEL_TENSOR.LOOP_MSG_QUERY: "blk.{bid}.loop_msg_query",
1907+
MODEL_TENSOR.LOOP_Q_CTRL_DOWN: "blk.{bid}.loop_q_ctrl_down",
1908+
MODEL_TENSOR.LOOP_Q_CTRL_GAIN: "blk.{bid}.loop_q_ctrl_gain",
1909+
MODEL_TENSOR.LOOP_Q_CTRL_FORGET: "blk.{bid}.loop_q_ctrl_forget",
1910+
MODEL_TENSOR.LOOP_K_CTRL_DOWN: "blk.{bid}.loop_k_ctrl_down",
1911+
MODEL_TENSOR.LOOP_K_CTRL_GAIN: "blk.{bid}.loop_k_ctrl_gain",
1912+
MODEL_TENSOR.LOOP_K_CTRL_FORGET: "blk.{bid}.loop_k_ctrl_forget",
1913+
MODEL_TENSOR.LOOP_V_CTRL_DOWN: "blk.{bid}.loop_v_ctrl_down",
1914+
MODEL_TENSOR.LOOP_V_CTRL_GAIN: "blk.{bid}.loop_v_ctrl_gain",
1915+
MODEL_TENSOR.LOOP_V_CTRL_FORGET: "blk.{bid}.loop_v_ctrl_forget",
1916+
MODEL_TENSOR.LOOP_FFN_CTRL_DOWN: "blk.{bid}.loop_ffn_ctrl_down",
1917+
MODEL_TENSOR.LOOP_FFN_CTRL_GAIN: "blk.{bid}.loop_ffn_ctrl_gain",
1918+
MODEL_TENSOR.LOOP_FFN_CTRL_FORGET: "blk.{bid}.loop_ffn_ctrl_forget",
18841919
MODEL_TENSOR.STREAM_EMBD: "stream_embd",
18851920
MODEL_TENSOR.DEPTH_MEM_STATE_WRITE: "depth_mem_state_write",
18861921
MODEL_TENSOR.DEPTH_MEM_META_WRITE: "depth_mem_meta_write",
@@ -7087,6 +7122,55 @@ class MODEL_TENSOR(IntEnum):
70877122
MODEL_TENSOR.FFN_POST_CONV,
70887123
MODEL_TENSOR.FFN_RES_QUERY,
70897124
],
7125+
MODEL_ARCH.INKLING2_LOOP: [
7126+
MODEL_TENSOR.TOKEN_EMBD,
7127+
MODEL_TENSOR.TOKEN_EMBD_NORM,
7128+
MODEL_TENSOR.OUTPUT,
7129+
MODEL_TENSOR.OUTPUT_NORM,
7130+
MODEL_TENSOR.OUTPUT_RES_QUERY,
7131+
MODEL_TENSOR.ATTN_NORM,
7132+
MODEL_TENSOR.ATTN_Q,
7133+
MODEL_TENSOR.ATTN_K,
7134+
MODEL_TENSOR.ATTN_V,
7135+
MODEL_TENSOR.ATTN_OUT,
7136+
MODEL_TENSOR.ATTN_GATE,
7137+
MODEL_TENSOR.ATTN_REL,
7138+
MODEL_TENSOR.ATTN_REL_BIAS,
7139+
MODEL_TENSOR.ATTN_Q_CLIP,
7140+
MODEL_TENSOR.ATTN_REL_CLIP,
7141+
MODEL_TENSOR.ATTN_SINK_PROJ,
7142+
MODEL_TENSOR.ATTN_Q_INP_NORM,
7143+
MODEL_TENSOR.ATTN_K_INP_NORM,
7144+
MODEL_TENSOR.ATTN_V_INP_NORM,
7145+
MODEL_TENSOR.ATTN_K_CONV,
7146+
MODEL_TENSOR.ATTN_V_CONV,
7147+
MODEL_TENSOR.ATTN_K_CONV_PROJ,
7148+
MODEL_TENSOR.ATTN_V_CONV_PROJ,
7149+
MODEL_TENSOR.ATTN_POST_CONV,
7150+
MODEL_TENSOR.ATTN_RES_QUERY,
7151+
MODEL_TENSOR.FFN_NORM,
7152+
MODEL_TENSOR.FFN_GATE_INP,
7153+
MODEL_TENSOR.FFN_EXP_PROBS_B,
7154+
MODEL_TENSOR.FFN_GATE_EXP,
7155+
MODEL_TENSOR.FFN_UP_EXP,
7156+
MODEL_TENSOR.FFN_GATE_UP_EXP,
7157+
MODEL_TENSOR.FFN_DOWN_EXP,
7158+
MODEL_TENSOR.FFN_POST_CONV,
7159+
MODEL_TENSOR.FFN_RES_QUERY,
7160+
MODEL_TENSOR.LOOP_MSG_QUERY,
7161+
MODEL_TENSOR.LOOP_Q_CTRL_DOWN,
7162+
MODEL_TENSOR.LOOP_Q_CTRL_GAIN,
7163+
MODEL_TENSOR.LOOP_Q_CTRL_FORGET,
7164+
MODEL_TENSOR.LOOP_K_CTRL_DOWN,
7165+
MODEL_TENSOR.LOOP_K_CTRL_GAIN,
7166+
MODEL_TENSOR.LOOP_K_CTRL_FORGET,
7167+
MODEL_TENSOR.LOOP_V_CTRL_DOWN,
7168+
MODEL_TENSOR.LOOP_V_CTRL_GAIN,
7169+
MODEL_TENSOR.LOOP_V_CTRL_FORGET,
7170+
MODEL_TENSOR.LOOP_FFN_CTRL_DOWN,
7171+
MODEL_TENSOR.LOOP_FFN_CTRL_GAIN,
7172+
MODEL_TENSOR.LOOP_FFN_CTRL_FORGET,
7173+
],
70907174
MODEL_ARCH.DOMYN_EDGE: [
70917175
MODEL_TENSOR.TOKEN_EMBD,
70927176
MODEL_TENSOR.ATTN_Q,

‎gguf-py/gguf/gguf_writer.py‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -990,6 +990,21 @@ def add_hrm_l_cycles(self, value: int) -> None:
990990
def add_hrm_prefix_lm(self, value: bool) -> None:
991991
self.add_bool(Keys.HRM.PREFIX_LM.format(arch=self.arch), value)
992992

993+
def add_loop_start(self, value: int) -> None:
994+
self.add_uint32(Keys.Loop.START.format(arch=self.arch), value)
995+
996+
def add_loop_end(self, value: int) -> None:
997+
self.add_uint32(Keys.Loop.END.format(arch=self.arch), value)
998+
999+
def add_loop_count(self, value: int) -> None:
1000+
self.add_uint32(Keys.Loop.COUNT.format(arch=self.arch), value)
1001+
1002+
def add_loop_gate_rank(self, value: int) -> None:
1003+
self.add_uint32(Keys.Loop.GATE_RANK.format(arch=self.arch), value)
1004+
1005+
def add_loop_varying(self, value: bool) -> None:
1006+
self.add_bool(Keys.Loop.VARYING.format(arch=self.arch), value)
1007+
9931008
def add_metis_pass_count(self, value: int) -> None:
9941009
self.add_uint32(Keys.Metis.PASS_COUNT.format(arch=self.arch), value)
9951010

‎gguf-py/gguf/tensor_mapping.py‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -543,6 +543,66 @@ class TensorNameMap:
543543
"model.layers.{bid}.attention_input.query", # inkling2
544544
),
545545

546+
MODEL_TENSOR.LOOP_Q_CTRL_DOWN: (
547+
"model.layers.{bid}.q_controller.down_proj", # inkling2_loop
548+
"model.layers.{bid}.controllers.attn.down_weight", # inkling2_loop
549+
),
550+
551+
MODEL_TENSOR.LOOP_Q_CTRL_GAIN: (
552+
"model.layers.{bid}.q_controller.gain_proj", # inkling2_loop
553+
"model.layers.{bid}.controllers.attn.gain_weight", # inkling2_loop
554+
),
555+
556+
MODEL_TENSOR.LOOP_Q_CTRL_FORGET: (
557+
"model.layers.{bid}.q_controller.forget_proj", # inkling2_loop
558+
"model.layers.{bid}.controllers.attn.forget_weight", # inkling2_loop
559+
),
560+
561+
MODEL_TENSOR.LOOP_K_CTRL_DOWN: (
562+
"model.layers.{bid}.k_controller.down_proj", # inkling2_loop
563+
"model.layers.{bid}.controllers.k.down_weight", # inkling2_loop
564+
),
565+
566+
MODEL_TENSOR.LOOP_K_CTRL_GAIN: (
567+
"model.layers.{bid}.k_controller.gain_proj", # inkling2_loop
568+
"model.layers.{bid}.controllers.k.gain_weight", # inkling2_loop
569+
),
570+
571+
MODEL_TENSOR.LOOP_K_CTRL_FORGET: (
572+
"model.layers.{bid}.k_controller.forget_proj", # inkling2_loop
573+
"model.layers.{bid}.controllers.k.forget_weight", # inkling2_loop
574+
),
575+
576+
MODEL_TENSOR.LOOP_V_CTRL_DOWN: (
577+
"model.layers.{bid}.v_controller.down_proj", # inkling2_loop
578+
"model.layers.{bid}.controllers.v.down_weight", # inkling2_loop
579+
),
580+
581+
MODEL_TENSOR.LOOP_V_CTRL_GAIN: (
582+
"model.layers.{bid}.v_controller.gain_proj", # inkling2_loop
583+
"model.layers.{bid}.controllers.v.gain_weight", # inkling2_loop
584+
),
585+
586+
MODEL_TENSOR.LOOP_V_CTRL_FORGET: (
587+
"model.layers.{bid}.v_controller.forget_proj", # inkling2_loop
588+
"model.layers.{bid}.controllers.v.forget_weight", # inkling2_loop
589+
),
590+
591+
MODEL_TENSOR.LOOP_FFN_CTRL_DOWN: (
592+
"model.layers.{bid}.mlp_controller.down_proj", # inkling2_loop
593+
"model.layers.{bid}.controllers.mlp.down_weight", # inkling2_loop
594+
),
595+
596+
MODEL_TENSOR.LOOP_FFN_CTRL_GAIN: (
597+
"model.layers.{bid}.mlp_controller.gain_proj", # inkling2_loop
598+
"model.layers.{bid}.controllers.mlp.gain_weight", # inkling2_loop
599+
),
600+
601+
MODEL_TENSOR.LOOP_FFN_CTRL_FORGET: (
602+
"model.layers.{bid}.mlp_controller.forget_proj", # inkling2_loop
603+
"model.layers.{bid}.controllers.mlp.forget_weight", # inkling2_loop
604+
),
605+
546606
# Feed-forward norm
547607
MODEL_TENSOR.FFN_NORM: (
548608
"layers.{bid}.moe_connection.norm", # metis16

0 commit comments

Comments
 (0)
⌂ 换网址