gaulthiergain-tools/srcs/extractertool/run_extractertool.go

package extractertool

import (
	"errors"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	u "tools/srcs/common"
)
import "C"

const HTTP = "http"
const URL = "URL"

type MicroLibFile struct {
	Functions []MicroLibsFunction `json:"functions"`
}

type MicroLibsFunction struct {
	Name string `json:"name"`
}

type Variables struct {
	name  string
	value string
}

func getMakefileSources(content string, mapSources map[string]string) {
	var re = regexp.MustCompile(`(?m)\/.*\.c|\/.*\.h|\/.*\.cpp`)
	for _, match := range re.FindAllString(content, -1) {
		vars := strings.Split(match, "/")
		mapSources[vars[len(vars)-1]] = match
	}
}

func findVariables(content string, mapVariables map[string]*Variables) {
	var re = regexp.MustCompile(`(?m)\$\([A-Z0-9_\-]*\)`)

	for _, match := range re.FindAllString(content, -1) {

		if _, ok := mapVariables[match]; !ok {

			v := &Variables{
				name:  match[2 : len(match)-1],
				value: "",
			}

			regexVar := regexp.MustCompile("(?m)" + v.name + "=.*$")
			for _, matchVar := range regexVar.FindAllString(content, -1) {
				v.value = matchVar
				break
			}

			mapVariables[match] = v
		}
	}
}

func resolveVariables(mapVariables map[string]*Variables) {
	for _, value := range mapVariables {

		var re = regexp.MustCompile(`(?m)\$\([A-Z0-9_\-]*\)`)

		resolved := false
		varString := ""
		for _, match := range re.FindAllString(value.value, -1) {
			vars := strings.Split(mapVariables[match].value, "=")
			if len(vars) > 1 {
				varString = vars[1]
			} else {
				varString = mapVariables[match].value
			}

			value.value = strings.Replace(value.value, match, varString, -1)
			resolved = true
		}
		if !resolved {
			vars := strings.Split(value.value, "=")
			if len(vars) > 1 {
				varString = vars[1]
			}
			value.value = varString
		}
	}
}

func detectURL(mapVariables map[string]*Variables) *string {
	for key, value := range mapVariables {
		if strings.Contains(key, URL) && strings.Contains(value.value, HTTP) {
			vars := strings.Split(value.value, "=")
			if len(vars) > 1 {
				return &vars[1]
			}
			return &value.value
		}
	}
	return nil
}

// TODO REPLACE
func CreateFolder(path string) (bool, error) {
	if _, err := os.Stat(path); os.IsNotExist(err) {
		if err = os.Mkdir(path, 0755); err != nil {
			return false, err
		}
		return true, nil
	}
	return false, nil
}

func DownloadFile(filepath string, url string) error {

	// Get the data
	resp, err := http.Get(url)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	// Create the file
	out, err := os.Create(filepath)
	if err != nil {
		return err
	}
	defer out.Close()

	// Write the body to file
	_, err = io.Copy(out, resp.Body)
	return err
}

func findSourcesFiles(workspace string) ([]string, error) {

	var filenames []string

	err := filepath.Walk(workspace,
		func(path string, info os.FileInfo, err error) error {
			if err != nil {
				return err
			}

			ext := filepath.Ext(info.Name())
			if ext == ".c" || ext == ".cpp" {
				filenames = append(filenames, path)
			}
			return nil
		})
	if err != nil {
		return nil, err
	}
	return filenames, nil
}

// TODO REPLACE
// ExecuteCommand a single command without displaying the output.
//
// It returns a string which represents stdout and an error if any, otherwise
// it returns nil.
func ExecuteCommand(command string, arguments []string) (string, error) {
	out, err := exec.Command(command, arguments...).CombinedOutput()
	if err != nil {
		return "", err
	}
	return string(out), nil
}

func saveSymbols(output string, mapSymbols map[string]string, libName string) {
	if strings.Contains(output, "\n") {
		output = strings.TrimSuffix(output, "\n")
	}

	symbols := strings.Split(output, ",")
	for _, s := range symbols {
		if len(s) > 0 {
			if _, ok := mapSymbols[s]; !ok {
				if s == "main" || strings.Contains(s, "test") {
					u.PrintWarning("Ignore function: " + s)
				} else {
					mapSymbols[s] = libName
				}
			}
		}
	}
}

func extractPrototype(sourcesFiltered []string, mapSymbols map[string]string, libName string) error {

	for _, f := range sourcesFiltered {
		script := filepath.Join(os.Getenv("GOPATH"), "src", "tools", "srcs", "extractertool", "parserClang.py")
		output, err := ExecuteCommand("python3", []string{script, "-q", "-t", f})
		if err != nil {
			u.PrintWarning("Incomplete analysis with file " + f)
			continue
		}
		saveSymbols(output, mapSymbols, libName)
	}
	return nil
}

func filterSourcesFiles(files []string, mapSources map[string]string) []string {
	var sourcesFiltered []string
	for _, f := range files {
		vars := strings.Split(f, "/")
		if len(vars) > 1 {
			filename := vars[len(vars)-1]
			if _, ok := mapSources[filename]; ok {
				sourcesFiltered = append(sourcesFiltered, f)
			}
		}
	}
	return sourcesFiltered
}

func RunExtracterTool(homeDir string) {

	// Init and parse local arguments
	args := new(u.Arguments)
	p, err := args.InitArguments("--extracter",
		"The extracter tool allows to extract all the symbols (functions) of an external/internal library")
	if err != nil {
		u.PrintErr(err)
	}
	if err := parseLocalArguments(p, args); err != nil {
		u.PrintErr(err)
	}

	var workspacePath = homeDir + u.SEP + u.WORKSPACEFOLDER
	if len(*args.StringArg[workspaceArg]) > 0 {
		workspacePath = *args.StringArg[workspaceArg]
	}

	libpath := *args.StringArg[library]
	lib := libpath
	if filepath.IsAbs(libpath) {
		lib = filepath.Base(libpath)
	} else {
		libpath = filepath.Join(workspacePath, u.LIBSFOLDER, lib)
	}

	file, err := ioutil.ReadFile(filepath.Join(libpath, "Makefile.uk"))
	if err != nil {
		u.PrintErr(err)
	}
	mapVariables := make(map[string]*Variables)
	content := string(file)

	mapSources := make(map[string]string)
	getMakefileSources(content, mapSources)

	findVariables(content, mapVariables)
	resolveVariables(mapVariables)
	url := detectURL(mapVariables)
	if url == nil {
		u.PrintErr(errors.New("url of the lib not found"))
		return
	}

	var fileExtension string
	urlSplit := strings.Split(*url, "/")
	if urlSplit[len(urlSplit)-1] == "download" {
		fileExtension = filepath.Ext(urlSplit[len(urlSplit)-2])
	} else {
		fileExtension = filepath.Ext(*url)
	}

	folderName := lib + "_sources_folder"
	created, err := CreateFolder(folderName)
	if err != nil {
		u.PrintErr(err)
	}

	var files []string
	var archiveName string

	if fileExtension == ".gz" {
		archiveName = lib + "_sources.tar" + fileExtension

	} else {
		archiveName = lib + "_sources" + fileExtension
	}

	if created {
		u.PrintInfo(*url + " is found. Download the lib sources...")
		err := DownloadFile(archiveName, *url)
		if err != nil {
			u.PrintErr(err)
		}
		u.PrintOk(*url + " successfully downloaded.")

		u.PrintInfo("Extracting " + archiveName + "...")
		if fileExtension == ".zip" {
			files, err = Unzip(archiveName, folderName)
			if err != nil {
				_ = os.Remove(archiveName)
				_ = os.RemoveAll(folderName)
				u.PrintErr(err.Error() + ". Corrupted archive. Please try again.")
			}

		} else if fileExtension == ".tar" || fileExtension == ".gz" || fileExtension == ".tgz" {
			files, err = unTarGz(archiveName, folderName)
			if err != nil {
				_ = os.Remove(archiveName)
				_ = os.RemoveAll(folderName)
				u.PrintErr(err.Error() + ". Corrupted archive. Please try again.")
			}

		} else {
			u.PrintErr(errors.New("unknown extension for archive"))
		}
	}

	if len(files) == 0 {
		u.PrintInfo("Inspecting folder " + folderName + " for sources...")
		files, err = findSourcesFiles(folderName)
		if err != nil {
			u.PrintErr(err)
		}
	}

	sourcesFiltered := filterSourcesFiles(files, mapSources)

	u.PrintInfo("Find " + strconv.Itoa(len(sourcesFiltered)) + " files to analyse")

	mapSymbols := make(map[string]string)
	u.PrintInfo("Extracting symbols from all sources of " + lib + ". This may take some times...")
	if err := extractPrototype(sourcesFiltered, mapSymbols, lib); err != nil {
		u.PrintErr(err)
	}

	mf := MicroLibFile{}
	mf.Functions = make([]MicroLibsFunction, len(mapSymbols))
	i := 0
	for k, _ := range mapSymbols {
		mf.Functions[i].Name = k
		i++
	}

	u.PrintOk(strconv.Itoa(len(mapSymbols)) + " symbols from " + lib + " have been extracted.")
	filename := filepath.Join(os.Getenv("GOPATH"), "src", "tools", "libs", "external", lib)
	if err := u.RecordDataJson(filename, mf); err != nil {
		u.PrintErr(err)
	} else {
		u.PrintOk("Symbols file have been written to " + filename + ".json")
	}

	u.PrintInfo("Remove folders " + archiveName + " and " + folderName)
	_ = os.Remove(archiveName)
	_ = os.RemoveAll(folderName)
}