stream

package
v0.0.0-...-9898e4f Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Oct 27, 2026 License: MIT Imports: 11 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

View Source
var Cmd = &cobra.Command{
	Use:   "stream",
	Short: "Stream data into a graph from Kafka",
	Long:  ``,
	Args:  cobra.ExactArgs(1),
	RunE: func(cmd *cobra.Command, args []string) error {

		kafkaHost := args[0]
		conn, err := gripql.Connect(rpc.ConfigWithDefaults(host), true)
		if err != nil {
			return err
		}
		defer conn.Close()

		config := sarama.NewConfig()
		config.Version = sarama.V2_8_0_0
		config.Net.SASL.Enable = true
		config.Net.SASL.User = KafkaUsername
		config.Net.SASL.Password = KafkaPassword
		config.Net.SASL.Mechanism = sarama.SASLTypePlaintext
		config.Producer.Return.Successes = true
		config.Net.SASL.Handshake = true
		config.Net.TLS.Enable = false
		if KafkaUsername == "" || KafkaPassword == "" || Topic == "" {
			return fmt.Errorf("Kafka username and password and topic must be populated")
		}

		consumer, err := sarama.NewConsumer([]string{kafkaHost}, config)
		if err != nil {
			return fmt.Errorf("failed to create Kafka consumer: %w", err)
		}
		defer consumer.Close()

		writeConsumer, err := consumer.ConsumePartition(Topic, 0, sarama.OffsetOldest)
		if err != nil {
			return fmt.Errorf("failed to consume partition: %w", err)
		}
		defer writeConsumer.Close()
		done := make(chan bool)

		go func() {
			defer close(done)
			for {

				timeout := time.NewTimer(3 * time.Second)
				select {
				case msg, ok := <-writeConsumer.Messages():
					if !ok {
						log.Info("writeConsumer channel closed")
						return
					}
					timeout.Stop()
					Method := ""
					Path := ""
					for _, HeaderMsg := range msg.Headers {
						if bytes.Equal(HeaderMsg.Key, []byte("PATH")) {
							Path = string(HeaderMsg.Value)
						} else if bytes.Equal(HeaderMsg.Key, []byte("METHOD")) {
							Method = string(HeaderMsg.Value)
						}
					}
					if Path == "" || Method == "" {
						continue
					}

					fmt.Println("PATH: ", Path, "METHOD: ", Method)
					bulk := regexp.MustCompile("^/v1/graph$")
					graph := regexp.MustCompile("^/v1/graph/([^/]+)$")
					switch Method {
					case "DELETE":
						deleteVertex := regexp.MustCompile("^/v1/graph/([^/]+)/vertex/([^/]+)$")
						deleteEdge := regexp.MustCompile("^/v1/graph/([^/]+)/edge/([^/]+)$")
						switch {
						case bulk.MatchString(Path):
							g := gripql.DeleteData{}
							err := protojson.Unmarshal(msg.Value, &g)
							if err != nil {
								log.WithFields(log.Fields{"error": err, "value": string(msg.Value)}).Error("consumer: bulk delete unmarshal vertex error")
								continue
							}
							err = conn.BulkDelete(&g)
							if err != nil {
								log.WithFields(log.Fields{"error": err, "path": Path}).Errorf("consumer: bulk delete")
								continue
							}
						case deleteVertex.MatchString(Path):
							id := strings.Split(Path, "/")
							err := conn.DeleteVertex(id[len(id)-3], id[len(id)-1])
							if err != nil {
								log.WithFields(log.Fields{"error": err, "path": Path}).Errorf("consumer: delete vertex")
								continue
							}
						case deleteEdge.MatchString(Path):
							id := strings.Split(Path, "/")
							err := conn.DeleteEdge(id[len(id)-3], id[len(id)-1])
							if err != nil {
								log.WithFields(log.Fields{"error": err, "path": Path}).Errorf("consumer: delete edge")
								continue
							}
						case graph.MatchString(Path):
							paths := strings.Split(Path, "/")
							graphName := paths[len(paths)-1]
							err = conn.DeleteGraph(graphName)
							if err != nil {
								log.WithFields(log.Fields{"error": err, "graph": graphName}).Errorf("consumer: delete graph")
							}
						default:
							continue
						}
					case "POST":
						addVertex := regexp.MustCompile("^/v1/graph/([^/]+)/vertex$")
						addEdge := regexp.MustCompile("^/v1/graph/([^/]+)/edge$")
						addSchema := regexp.MustCompile("^/v1/graph/([^/]+)/schema$")
						switch {
						case bulk.MatchString(Path):
							fmt.Printf("WE HERE: %s", string(msg.Value))
						case addVertex.MatchString(Path):
							v := gripql.Vertex{}
							err = protojson.Unmarshal(msg.Value, &v)
							if err != nil {
								log.WithFields(log.Fields{"error": err, "value": string(msg.Value)}).Error("consumer: unmarshal vertex error")
								continue
							}
							id := strings.Split(Path, "/")
							err = conn.AddVertex(id[len(id)-2], &v)
							if err != nil {
								log.WithFields(log.Fields{"error": err, "vertex": v}).Error("consumer: add vertex error")
								continue
							}
						case addEdge.MatchString(Path):
							e := gripql.Edge{}
							err = protojson.Unmarshal(msg.Value, &e)
							if err != nil {
								log.WithFields(log.Fields{"error": err, "value": string(msg.Value)}).Error("consumer: unmarshal edge error")
								continue
							}
							id := strings.Split(Path, "/")
							err = conn.AddEdge(id[len(id)-2], &e)
							if err != nil {
								log.WithFields(log.Fields{"error": err, "edge": e}).Error("consumer: add edge error")
								continue
							}
						case addSchema.MatchString(Path):
							graph := gripql.Graph{}
							err = protojson.Unmarshal(msg.Value, &graph)
							if err != nil {
								log.WithFields(log.Fields{"error": err, "value": string(msg.Value)}).Error("consumer: unmarshal schema error")
								continue
							}
							err = conn.AddSchema(&graph)
							if err != nil {
								log.WithFields(log.Fields{"error": err, "schema": graph}).Error("consumer: add schema error")
							}
						case graph.MatchString(Path):
							paths := strings.Split(Path, "/")
							graphName := paths[len(paths)-1]
							err = conn.AddGraph(graphName)
							if err != nil {
								log.WithFields(log.Fields{"error": err, "graph": graphName}).Error("consumer: add graph error")
							}
						default:
							continue
						}
					default:
						continue
					}
				case <-timeout.C:
					return
				}
			}
		}()

		<-done
		return nil
	},
}

Fetch all requests from Kafka and recreate all write Ex command using ci env: grip stream localhost:9092 --KafkaUsername admin --KafkaPassword adminpassword --Topic gripHistory

View Source
var KafkaPassword = ""
View Source
var KafkaUsername = ""
View Source
var Topic = ""

Functions

This section is empty.

Types

This section is empty.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
⌂ 换网址