Programming and Exciting Things

MySQL usage of new JSON field

Published on 25.12.2015

As a part from my new project, I'm using MySQL JSON field which was introduced in version 5.7, functions are described in https://dev.mysql.com/doc/refman/5.7/en/json-functions.html Creating JSON data is pretty simple like:

CREATE TABLE `events` (
	`id` INT(11) NOT NULL AUTO_INCREMENT,
	`site_id` INT(11) NOT NULL,
	`category_id` INT(11) NOT NULL,
	`name` VARCHAR(250) NOT NULL,
	`value` TEXT NULL,
	`added_time` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
	`custom_data` JSON NULL DEFAULT NULL,
	PRIMARY KEY (`id`),
	INDEX `site_id` (`site_id`, `category_id`)
)
COLLATE='utf8_general_ci'
ENGINE=InnoDB
AUTO_INCREMENT=0;
As you can see Create Table syntax is same, difference is in
 `custom_data` JSON 
. So let's insert some data to it.
INSERT INTO `events` (`id`, `site_id`, `category_id`, `name`, `value`, `added_time`, `custom_data`) VALUES (3, 1, 4, 'fsafsa', '21', '2015-12-21 23:05:56', NULL);
INSERT INTO `events` (`id`, `site_id`, `category_id`, `name`, `value`, `added_time`, `custom_data`) VALUES (4, 1, 4, 'fsafsa', '21', '2015-12-21 23:06:09', NULL);
INSERT INTO `events` (`id`, `site_id`, `category_id`, `name`, `value`, `added_time`, `custom_data`) VALUES (5, 1, 4, 'fsafsa', '2кири', '2015-12-21 23:06:16', NULL);
INSERT INTO `events` (`id`, `site_id`, `category_id`, `name`, `value`, `added_time`, `custom_data`) VALUES (6, 1, 4, 'fsafsa', '2кири', '2015-12-25 10:11:54', '{"dada": {"rpb": "22"}, "test": "dada"}');
INSERT INTO `events` (`id`, `site_id`, `category_id`, `name`, `value`, `added_time`, `custom_data`) VALUES (7, 1, 1, 'test', '1', '2015-12-25 17:26:04', NULL);
INSERT INTO `events` (`id`, `site_id`, `category_id`, `name`, `value`, `added_time`, `custom_data`) VALUES (8, 1, 1, '1da', '2', '2015-12-25 17:26:10', '{"test": "dada"}');

And lets assume that we want to filter data in JSON field.
SELECT  id,custom_data FROM `events` WHERE JSON_UNQUOTE(custom_data->"$.test") = "dada"
Result is:
+----+-----------------------------------------+
| id | custom_data                             |
+----+-----------------------------------------+
|  6 | {"dada": {"rpb": "22"}, "test": "dada"} |
|  8 | {"test": "dada"}                        |
+----+-----------------------------------------+
2 rows in set (0.00 sec)
Or if we want to check existence of sub element:
SELECT id,custom_data  FROM `events` WHERE  JSON_CONTAINS_PATH(custom_data,"all","$.dada.rpb");
Result is:
+----+-----------------------------------------+
| id | custom_data                             |
+----+-----------------------------------------+
|  6 | {"dada": {"rpb": "22"}, "test": "dada"} |
+----+-----------------------------------------+
1 row in set (0.00 sec)
And one more example for extracting value from JSON:
SELECT 
	id,
	JSON_UNQUOTE(JSON_EXTRACT(custom_data,'$.test')) AS value
FROM `events`
WHERE JSON_EXTRACT(custom_data,'$.test') IS NOT NULL
+----+-------+
| id | value |
+----+-------+
|  6 | dada  |
|  8 | dada  |
+----+-------+
2 rows in set (0.00 sec)

It's look pretty easy, right ? :)

Go lang create map from database results without using predefined struct

Published on 07.12.2015

I created some function which get "random" data from MySQL database and return it as JSON. Here is some example code for achieve this, I skipped boring part with imports and creating net/http server and error handling is skipped.


db, _ := sql.Open(
        "mysql",
        "user=demo dbname=demo password=test")
		
rows, _ := db.Query("select * from sites")

columns, _ := rows.Columns()
count := len(columns)
values := make([]interface{}, count)
valuePtrs := make([]interface{}, count)

final_result := map[int]map[string]string{}
result_id := 0
for rows.Next() {
	for i, _ := range columns {
		valuePtrs[i] = &values[i]
	}
	rows.Scan(valuePtrs...)

	tmp_struct := map[string]string{}

	for i, col := range columns {
		var v interface{}
		val := values[i]
		b, ok := val.([]byte)
		if (ok) {
			v = string(b)
		} else {
			v = val
		}
		tmp_struct[col] = fmt.Sprintf("%s",v)
	}
	
	final_result[ result_id ] = tmp_struct
	result_id++
}

fmt.Println(final_result)

Fake rsyslog server with Go Lang

Published on 18.11.2015

Few months ago I created script which act as fake rsyslog server with python, so because I fall in love with Go Lang, here is updated version of this script with added logging into MongoDB

package main

import (
	"encoding/json"
	"fmt"
	"github.com/manucorporat/try"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
	"net"
	"os"
	"time"
)

const (
	MongoDBHosts      = "127.0.0.1:27017"
	MongoAuthUserName = "root"
	MongoAuthPassword = ""
)

type ParsedMsg struct {
	Msg     string
	ApiKey  string `json:"api_key"`
	AppName string `json:"app-name"`
}

type MongoDoc struct {
	ID        bson.ObjectId `bson:"_id,omitempty"`
	Message   string
	Program   string
	ApiKey    string
	Timestamp time.Time
}


func getSession() *mgo.Session {

	mongoDBDialInfo := &mgo.DialInfo{
		Addrs:   []string{MongoDBHosts},
		Timeout: 60 * time.Second,
	}

	// Connect to our local mongo
	s, err := mgo.DialWithInfo(mongoDBDialInfo)

	// Check if connection error, is mongo running?
	if err != nil {
		panic(err)
	}
	s.SetMode(mgo.Monotonic, true)

	return s
}

func ParseMsg(str string, mongoSession *mgo.Session) {

	var j map[string]interface{}
	err := json.Unmarshal([]byte(str), &j)
	if err != nil {
		panic(err)
	}

	res := ParsedMsg{}
	json.Unmarshal([]byte(str), &res)

	c := mongoSession.DB("logs").C(res.AppName)

	new_doc := &MongoDoc{ID: bson.NewObjectId(), Message: res.Msg, Program: res.AppName, ApiKey: res.ApiKey, Timestamp: time.Now()}

	c.Insert(new_doc)
	if err != nil {
		fmt.Println(err)
	}

	// inserted mongo id
	// string(new_doc.ID.Hex())

}

/* A Simple function to verify error */
func CheckError(err error) {
	if err != nil {
		fmt.Println("Error: ", err)
		os.Exit(0)
	}
}

func main() {
	/* Lets prepare a address at any address at port 10001*/
	ServerAddr, err := net.ResolveUDPAddr("udp", ":514")
	CheckError(err)

	/* Now listen at selected port */
	ServerConn, err := net.ListenUDP("udp", ServerAddr)
	CheckError(err)
	defer ServerConn.Close()

	mongoSession := getSession()

	buf := make([]byte, 1024)

	for {
		n, _, err := ServerConn.ReadFromUDP(buf)

		try.This(func() {
			ParseMsg(string(buf[0:n]), mongoSession)
		})

		if err != nil {
			fmt.Println("Error: ", err)
		}
	}
}

PHP Singlethon implementation

Published on 12.11.2015

Example of quick singlethon implementation in PHP

<?php 
abstract class Singleton {
    
    // protected static $INSTANCE;

   
    /**
     * Protected constructor to prevent creating a new instance of the
     * @return void
     */
    final private function __construct() {
    }

    /**
     * Private clone method to prevent cloning of the instance
     * @return void
     */
    final private function __clone() {
    }
    
    /**
     * Private unserialize method to prevent unserializing 
     * @return void
     */
    final private function __wakeup()
    {
    }

    /**
     * The reference to *Singleton* instance of this class
     * @return Singleton
     */
    final public static function getInstance() {
        return isset(static::$INSTANCE) ? static::$INSTANCE : static::$INSTANCE = new static;
    }

}

////

class AB extends Singleton {
    protected static $INSTANCE;
}

class BA extends Singleton {
    protected static $INSTANCE;
}

$obj = AB::getInstance();

$obj2 = BA::getInstance();


var_dump($obj === AB::getInstance()); // bool true
var_dump($obj === $obj2); // bool false

Availible also in https://gist.github.com/yuksbg/9d981ea6baf0bd8f2d92

Mysql to MongoDB exporter

Published on 02.11.2015

I just finished with simple python class for exporting MySQL tables to MongoDB
It can be use as part of project or as saparate command line application.

More information can be found at https://yuksbg.github.io/mysql2mongo/