kbruen/gaulthiergain-tools
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #1 from gaulthiergain/perski/dev

Browse source 
Perski/dev
This commit is contained in:
Gaulthier Gain 2023-07-13 21:09:14 +02:00 • committed by GitHub
commit 43f422e3d3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
16 changed files with 1342 additions and 157 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
srcs/Makefile
View file

@ -36,7 +36,7 @@ deps:
$(GOGET) github.com/akamensky/argparse
$(GOGET) github.com/awalterschulze/gographviz
$(GOGET) github.com/sergi/go-diff/...
$(GOGET) github.com/AlecAivazis/survey
$(GOGET) gopkg.in/AlecAivazis/survey.v1
# Cross compilation
build-linux:
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 $(GOBUILD) -o $(BINARY_UNIX) -v

13
srcs/buildtool/args.go
View file

@ -7,9 +7,10 @@
package buildtool
import (
"github.com/akamensky/argparse"
"os"
u "tools/srcs/common"
"github.com/akamensky/argparse"
)
const (
@ -18,6 +19,8 @@ const (
sourcesArg = "sources"
objsArg = "objects"
makefileArg = "makefile"
configArg = "config"
patchArg = "patch"
)
// ParseArguments parses arguments of the application.
@ -34,11 +37,15 @@ func parseLocalArguments(p *argparse.Parser, args *u.Arguments) error {
&argparse.Options{Required: true, Help: "App Sources " +
"Folder"})
args.InitArgParse(p, args, u.BOOL, "o", objsArg,
&argparse.Options{Required: false, Default: false, Help: "Add objects from external build system " +
"Folder"})
&argparse.Options{Required: false, Default: false, Help: "Add objects from external" +
"build system Folder"})
args.InitArgParse(p, args, u.STRING, "m", makefileArg,
&argparse.Options{Required: false, Help: "Add additional properties " +
"for Makefile"})
args.InitArgParse(p, args, u.STRINGLIST, "c", configArg,
&argparse.Options{Required: false, Help: "Add configuration files"})
args.InitArgParse(p, args, u.STRING, "", patchArg,
&argparse.Options{Required: false, Help: "Add patch files"})
return u.ParserWrapper(p, os.Args)
}

72
srcs/buildtool/microlibs_process.go
View file

@ -41,7 +41,6 @@ type MicroLibsFunction struct {
// -----------------------------Match micro-libs--------------------------------
// processSymbols adds symbols within the 'exportsyms.uk' file into a map.
//
func processSymbols(microLib, output string, mapSymbols map[string][]string) {
lines := strings.Split(output, "\n")
@ -134,6 +133,53 @@ func fetchSymbolsExternalLibs(folder string,
return externalLibs, nil
}
// putJsonSymbolsTogether puts the json file symbols and system calls resulting from the static,
// dynamic and source files analyses together into a map structure.
//
// It returns the map containing all the symbols and system calls.
func putJsonSymbolsTogether(data *u.Data) map[string]string {
dataMap := make(map[string]string)
for k, v := range data.StaticData.Symbols {
dataMap[k] = v
}
for k := range data.StaticData.SystemCalls {
dataMap[k] = ""
}
for k, v := range data.DynamicData.Symbols {
dataMap[k] = v
}
for k := range data.DynamicData.SystemCalls {
dataMap[k] = ""
}
for k, v := range data.SourcesData.Symbols {
dataMap[k] = v
}
for k := range data.SourcesData.SystemCalls {
dataMap[k] = ""
}
return dataMap
}
// retNameCompat modifies its string argument in order to replace its underscore by a dash when
// necessary.
//
// It returns its string argument whose underscore has been replaced by a dash if necessary,
// otherwise it returns its argument unchanged.
func retNameForCompat(value string) string {
if strings.Contains(value, "posix-") {
return strings.ReplaceAll(value, "posix-", "posix_")
}
return value
}
// matchSymbols performs the matching between Unikraft's micro-libs and
// libraries used by a given application based on the list of symbols that both
// contain.
@ -144,15 +190,8 @@ func matchSymbols(matchedLibs []string, data map[string]string,
for key := range data {
if values, ok := microLibs[key]; ok {
for _, value := range values {
// todo remove
if strings.Compare(NOLIBC, value) == 0 {
value = NEWLIB
}
// remove above
if !u.Contains(matchedLibs, value) {
matchedLibs = append(matchedLibs, value)
if !u.Contains(matchedLibs, retNameForCompat(value)) {
matchedLibs = append(matchedLibs, retNameForCompat(value))
}
}
}
@ -172,11 +211,6 @@ func matchLibs(unikraftLibs string, data *u.Data) ([]string, map[string]string,
matchedLibs := make([]string, 0)
//todo remove
matchedLibs = append(matchedLibs, POSIXLIBDL)
matchedLibs = append(matchedLibs, POSIXSYSINFO)
matchedLibs = append(matchedLibs, UKMMAP)
folder := filepath.Join(os.Getenv("GOPATH"), "src", "tools", "libs", "internal")
if err := fetchSymbolsInternalLibs(folder, mapSymbols); err != nil {
return nil, nil, err
@ -189,11 +223,10 @@ func matchLibs(unikraftLibs string, data *u.Data) ([]string, map[string]string,
return nil, nil, err
}
// Perform the matching symbols on static data
matchedLibs = matchSymbols(matchedLibs, data.StaticData.Symbols, mapSymbols)
dataMap := putJsonSymbolsTogether(data)
// Perform the matching symbols on dynamic data
matchedLibs = matchSymbols(matchedLibs, data.DynamicData.Symbols, mapSymbols)
// Perform the symbol matching
matchedLibs = matchSymbols(matchedLibs, dataMap, mapSymbols)
return matchedLibs, externalLibs, nil
}
@ -223,7 +256,6 @@ func cloneGitRepo(url, unikraftPathLibs, lib string) error {
// cloneLibsFolders clones all the needed micro-libs that are needed by a
// given application
//
func cloneLibsFolders(workspacePath string, matchedLibs []string,
externalLibs map[string]string) {

95
srcs/buildtool/run_buildtool.go
View file

@ -7,13 +7,14 @@
package buildtool
import (
"github.com/AlecAivazis/survey/v2"
"io/ioutil"
"os"
"path/filepath"
"regexp"
"strings"
u "tools/srcs/common"
"gopkg.in/AlecAivazis/survey.v1"
)
// STATES
@ -51,6 +52,7 @@ func generateConfigUk(filename, programName string, matchedLibs []string) error
// execution of the 'make' command.
//
// It returns an integer that defines the result of 'make':
//
// <SUCCESS, LINKING_ERROR, COMPILER_ERROR>
func checkMakeOutput(appFolder string, stderr *string) int {
@ -103,11 +105,38 @@ func parseMakeOutput(output string) string {
return sb.String()
}
// addConfigFiles adds user-provided configuration files in the order in which they come to the
// unikernel folder.
func addConfigFiles(configFiles []string, selectedFiles *[]string, includeFolder,
appFolder string) {
for _, configFilePath := range configFiles {
configFile := filepath.Base(configFilePath)
fileExt := filepath.Ext(configFile)
if fileExt == ".h" || fileExt == ".hpp" || fileExt == ".hcc" {
if err := u.CopyFileContents(configFilePath, includeFolder+configFile); err != nil {
u.PrintErr(err)
}
} else if fileExt == ".c" || fileExt == ".cpp" || fileExt == ".cc" {
if err := u.CopyFileContents(configFilePath, appFolder+configFile); err != nil {
u.PrintErr(err)
}
// Add Makefile.uk entry
*selectedFiles = append(*selectedFiles, configFile)
} else {
if err := u.CopyFileContents(configFilePath, appFolder+configFile); err != nil {
u.PrintErr(err)
}
}
}
}
// -------------------------------------Run-------------------------------------
// RunBuildTool runs the automatic build tool to build a unikernel of a
// given application.
//
func RunBuildTool(homeDir string, data *u.Data) {
// Init and parse local arguments
@ -195,7 +224,7 @@ func RunBuildTool(homeDir string, data *u.Data) {
}
// Filter source files to limit build errors (e.g., remove test files,
//multiple main file, ...)
// multiple main file, ...)
filterSourceFiles := filterSourcesFiles(sourceFiles)
// Prompt file selection
@ -207,10 +236,31 @@ func RunBuildTool(homeDir string, data *u.Data) {
}
var selectedFiles []string
if err := survey.AskOne(prompt, &selectedFiles); err != nil {
if err := survey.AskOne(prompt, &selectedFiles, nil); err != nil {
panic(err)
}
// Copy, conform and apply user-provided patches to the new unikernel folder
if len(*args.StringArg[patchArg]) > 0 {
// Create patch folder
patchFolder, err := createPatchFolder(appFolder)
if err != nil {
u.PrintErr(err)
}
if err := addAndApplyPatchFiles(*args.StringArg[patchArg], *patchFolder, appFolder); err !=
nil {
u.PrintErr(err)
}
}
// Conform file include directives to the new unikernel folder organisation
if err := conformIncludeDirectives(appFolder); err != nil {
u.PrintErr(err)
}
// Add config files to the unikernel folder
addConfigFiles(*args.StringListArg[configArg], &selectedFiles, *includeFolder, appFolder)
// Match micro-libs
matchedLibs, externalLibs, err := matchLibs(unikraftPath+"lib"+u.SEP, data)
if err != nil {
@ -249,15 +299,35 @@ func RunBuildTool(homeDir string, data *u.Data) {
runMake(programName, appFolder)
}
// retFolderCompat modifies its string argument in order to replace its underscore by a dash when
// necessary for the searchInternalDependencies function to find the corresponding folder in the
// 'unikraft' folder.
//
// It returns its string argument whose underscore has been replaced by a dash if necessary,
// otherwise it returns its argument unchanged.
func retFolderForCompat(lib string) string {
if strings.Contains(lib, "posix_") {
return strings.ReplaceAll(lib, "posix_", "posix-")
}
return lib
}
func searchInternalDependencies(unikraftPath string, matchedLibs *[]string,
externalLibs map[string]string) error {
for _, lib := range *matchedLibs {
// Consider only external libs
if _, ok := externalLibs[lib]; ok {
// Get and read Config.UK from external lib
configUk := unikraftPath + u.LIBSFOLDER + lib + u.SEP + "Config.uk"
// Get and read Config.UK from lib
var configUk string
if _, ok := externalLibs[lib]; ok {
configUk = unikraftPath + u.LIBSFOLDER + lib + u.SEP + "Config.uk"
} else {
configUk = unikraftPath + u.UNIKRAFTFOLDER + "lib" + u.SEP + retFolderForCompat(lib) +
u.SEP + "Config.uk"
}
lines, err := u.ReadLinesFile(configUk)
if err != nil {
return err
@ -265,8 +335,7 @@ func searchInternalDependencies(unikraftPath string, matchedLibs *[]string,
// Process Config.UK file
mapConfig := make(map[string][]string)
u.ProcessConfigUK(lines, false, mapConfig, nil)
u.ProcessConfigUK(lines, true, mapConfig, nil)
for config := range mapConfig {
// Remove LIB prefix
@ -274,11 +343,6 @@ func searchInternalDependencies(unikraftPath string, matchedLibs *[]string,
config = strings.TrimPrefix(config, "LIB")
}
// Replace underscore by dash
if strings.Contains(config, "_") {
config = strings.ReplaceAll(config, "_", "-")
}
// Check if matchedLibs already contains the lib
config = strings.ToLower(config)
if !u.Contains(*matchedLibs, config) {
@ -286,7 +350,6 @@ func searchInternalDependencies(unikraftPath string, matchedLibs *[]string,
}
}
}
}
return nil
}

290
srcs/buildtool/unikraft_files_process.go
View file

@ -12,11 +12,24 @@ import (
"fmt"
"os"
"path/filepath"
"regexp"
"strings"
u "tools/srcs/common"
)
// ---------------------------Create Patches Folder-----------------------------
func createPatchFolder(appFolder string) (*string, error) {
patchesFolder := appFolder + "patches" + u.SEP
if _, err := u.CreateFolder(patchesFolder); err != nil {
return nil, err
}
return &patchesFolder, nil
}
// ---------------------------Create Include Folder-----------------------------
func createIncludeFolder(appFolder string) (*string, error) {
@ -109,7 +122,10 @@ func createUnikraftApp(programName, workspacePath string) (*string, error) {
if !created {
u.PrintWarning(appFolder + " already exists.")
handleCreationApp(&appFolder)
appFolder = handleCreationApp(appFolder)
if _, err := u.CreateFolder(appFolder); err != nil {
return nil, err
}
}
return &appFolder, nil
@ -117,24 +133,24 @@ func createUnikraftApp(programName, workspacePath string) (*string, error) {
// -----------------------------Create App folder-------------------------------
func handleCreationApp(appFolder *string) {
func handleCreationApp(appFolder string) string {
fmt.Println("Make your choice:\n1: Copy and overwrite files\n2: " +
"Enter manually the name of the folder\n3: exit program")
var input int
for true {
for {
fmt.Print("Please enter your choice (0 to exit): ")
if _, err := fmt.Scanf("%d", &input); err != nil {
u.PrintWarning("Choice must be numeric! Try again")
} else {
switch input {
case 1:
return
return appFolder
case 2:
fmt.Print("Enter text: ")
reader := bufio.NewReader(os.Stdin)
text, _ := reader.ReadString('\n')
appFolder = &text
return
appFolder = strings.Split(text, "\n")[0] + u.SEP
return appFolder
case 3:
os.Exit(1)
default:
@ -150,11 +166,11 @@ var srcLanguages = map[string]int{
".c": 0,
".cpp": 0,
".cc": 0,
".S": 0,
/*".S": 0,
".s": 0,
".asm": 0,
".py": 0,
".go": 0,
".go": 0,*/
}
func filterSourcesFiles(sourceFiles []string) []string {
@ -170,14 +186,264 @@ func filterSourcesFiles(sourceFiles []string) []string {
return filterSrcFiles
}
// addAndApplyPatchFiles copies all the user-provided patch files to the unikernel directory,
// conforms them to the unikernel directory format so that all paths in the patch files are paths
// to source files located in the unikernel folder and applies the patches.
//
// It returns an error if any, otherwise it returns nil.
func addAndApplyPatchFiles(patchPath string, patchFolder, appFolder string) error {
// Copy and conform patch files
err := filepath.Walk(patchPath, func(filePath string, info os.FileInfo,
err error) error {
if !info.IsDir() {
extension := filepath.Ext(info.Name())
if extension == ".patch" {
if err = conformPatchFile(filePath, patchFolder+info.Name()); err != nil {
return err
}
}
}
return nil
})
if err != nil {
return err
}
// Initialise git repo to be able to apply patches
_, _, _ = u.ExecuteRunCmd("git", appFolder, true, "init")
_, _, _ = u.ExecuteRunCmd("git", appFolder, true, "add", ".")
_, _, _ = u.ExecuteRunCmd("git", appFolder, true, "commit", "-m", "first commit")
// Apply patches
err = filepath.Walk(patchPath, func(filePath string, info os.FileInfo,
err error) error {
if !info.IsDir() {
_, _, _ = u.ExecuteRunCmd("git", appFolder, true, "am", patchFolder+info.Name())
}
return nil
})
if err != nil {
return err
}
return nil
}
// conformPatchFile copies all the user-provided patch files to the unikernel directory and
// conforms them to the unikernel directory format so that all paths in the patch files are paths
// to source files located in the unikernel folder.
//
// It returns an error if any, otherwise it returns nil.
func conformPatchFile(patchPath, newPatchPath string) error {
patchLines, err := u.ReadLinesFile(patchPath)
if err != nil {
return err
}
// Find paths in patch file using regexp
re1 := regexp.MustCompile(`( )(.*)( \| )(.*)`)
re2 := regexp.MustCompile(`(diff --git )(a/)?(.*)( )(b/)?(.*)`)
re3 := regexp.MustCompile(`(--- )(a/)?(.*)`)
re4 := regexp.MustCompile(`(\+\+\+ )(b/)?(.*)`)
for lineIndex := range patchLines {
// All paths to files to be modified by the patch are listed under "---"
if patchLines[lineIndex] == "---\n" {
lineIndex++
for ; !strings.Contains(patchLines[lineIndex], "changed"); lineIndex++ {
for _, match := range re1.FindAllStringSubmatch(patchLines[lineIndex], -1) {
conformPatchPath(&match, 2)
patchLines[lineIndex] = strings.Join(match[1:], "") + "\n"
}
}
}
// All diff lines contain paths to files to be modified by the patch
if len(patchLines[lineIndex]) > 10 && patchLines[lineIndex][:10] == "diff --git" {
for _, match := range re2.FindAllStringSubmatch(patchLines[lineIndex], -1) {
conformPatchPath(&match, 3)
conformPatchPath(&match, 6)
patchLines[lineIndex] = strings.Join(match[1:], "") + "\n"
}
// Same observation for the two lines following the index line
for _, match := range re3.FindAllStringSubmatch(patchLines[lineIndex+2], -1) {
conformPatchPath(&match, 3)
patchLines[lineIndex+2] = strings.Join(match[1:], "") + "\n"
}
for _, match := range re4.FindAllStringSubmatch(patchLines[lineIndex+3], -1) {
conformPatchPath(&match, 3)
patchLines[lineIndex+3] = strings.Join(match[1:], "") + "\n"
}
}
}
// Write the modified content to a file in the unikernel folder
err = u.WriteToFile(newPatchPath, []byte(strings.Join(patchLines, "")))
if err != nil {
return err
}
return nil
}
// conformPatchPath conforms a path in a user-provided patch file to the unikernel directory format
// so that this path describes a source file located in the unikernel folder.
func conformPatchPath(match *[]string, index int) {
extension := filepath.Ext((*match)[index])
if extension == ".h" || extension == ".hpp" || extension == ".hcc" {
(*match)[index] = "include" + u.SEP + filepath.Base((*match)[index])
} else {
(*match)[index] = filepath.Base((*match)[index])
}
}
// conformIncludeDirectives conforms all the user-defined include directives of all C/C++ source
// files to the unikernel directory format so that all these directives are paths to source files
// located in the include folder of the unikernel directory.
//
// It returns an error if any, otherwise it returns nil.
func conformIncludeDirectives(sourcePath string) error {
err := filepath.Walk(sourcePath, func(path string, info os.FileInfo,
err error) error {
if !info.IsDir() {
extension := filepath.Ext(info.Name())
if extension == ".h" || extension == ".hpp" || extension == ".hcc" {
if err = conformFile(path, true); err != nil {
return err
}
} else if extension == ".c" || extension == ".cpp" || extension == ".cc" {
if err = conformFile(path, false); err != nil {
return err
}
}
}
return nil
})
if err != nil {
return err
}
return nil
}
// conformFile conforms all the user-defined include directives of a C/C++ source file to the
// unikernel directory format so that all these directives are paths to source files located in the
// include folder.
//
// It returns an error if any, otherwise it returns nil.
func conformFile(filePath string, isHeader bool) (err error) {
fileLines, err := u.ReadLinesFile(filePath)
if err != nil {
return err
}
// Find include directives using regexp
re := regexp.MustCompile(`(.*)(#include)(.*)("|<)(.*)("|>)(.*)`)
for lineIndex := range fileLines {
for _, match := range re.FindAllStringSubmatch(fileLines[lineIndex], -1) {
// Only interested in included files not coming from the standard library
if incDirContainsStdFold(fileLines[lineIndex]) {
break
}
for i := 1; i < len(match); i++ {
if match[i] == "\"" || match[i] == "<" {
// Determine the included source file extension to know what the path to it
// must be in the current file
var extIsHeader bool
extension := filepath.Ext(match[i+1])
if extension == ".h" || extension == ".hpp" || extension == ".hcc" {
extIsHeader = true
} else if extension == ".c" || extension == ".cpp" || extension == ".cc" {
extIsHeader = false
} else {
// C++ header
break
}
// Modify the include path
match[i] = "\""
if isHeader && !extIsHeader {
match[i+1] = "../" + filepath.Base(match[i+1])
} else if !isHeader && extIsHeader {
match[i+1] = "include" + u.SEP + filepath.Base(match[i+1])
} else {
match[i+1] = filepath.Base(match[i+1])
}
match[i+2] = "\""
fileLines[lineIndex] = strings.Join(match[1:], "") + "\n"
break
}
}
break
}
}
// Write the modified content to a file in the unikernel folder
err = u.WriteToFile(filePath, []byte(strings.Join(fileLines, "")))
if err != nil {
return err
}
return nil
}
// incDirContainsStdFold determines if an include directive is a path to a standard header.
//
// It returns true if it is the case, false otherwise.
func incDirContainsStdFold(fileLine string) bool {
// Standard header list
stdHeaders := []string{
"<aio.h>", "<libgen.h>", "<spawn.h>", "<sys/time.h>",
"<arpa/inet.h>", "<limits.h>", "<stdarg.h>", "<sys/times.h>",
"<assert.h>", "<locale.h>", "<stdbool.h>", "<sys/types.h>",
"<complex.h>", "<math.h>", "<stddef.h>", "<sys/uio.h>",
"<cpio.h>", "<monetary.h>", "<stdint.h>", "<sys/un.h>",
"<ctype.h>", "<mqueue.h>", "<stdio.h>", "<sys/utsname.h>",
"<dirent.h>", "<ndbm.h>", "<stdlib.h>", "<sys/wait.h>",
"<dlfcn.h>", "<net/if.h>", "<string.h>", "<syslog.h>",
"<errno.h>", "<netdb.h>", "<strings.h>", "<tar.h>",
"<fcntl.h>", "<netinet/in.h>", "<stropts.h>", "<termios.h>",
"<fenv.h>", "<netinet/tcp.h>", "<sys/ipc.h>", "<tgmath.h>",
"<float.h>", "<nl_types.h>", "<sys/mman.h>", "<time.h>",
"<fmtmsg.h>", "<poll.h>", "<sys/msg.h>", "<trace.h>",
"<fnmatch.h>", "<pthread.h>", "<sys/resource.h>", "<ulimit.h>",
"<ftw.h>", "<pwd.h>", "<sys/select.h>", "<unistd.h>",
"<glob.h>", "<regex.h>", "<sys/sem.h>", "<utime.h>",
"<grp.h>", "<sched.h>", "<sys/shm.h>", "<utmpx.h>",
"<iconv.h>", "<search.h>", "<sys/socket.h>", "<wchar.h>",
"<inttypes.h>", "<semaphore.h>", "<sys/stat.h>", "<wctype.h>",
"<iso646.h>", "<setjmp.h>", "<sys/statvfs.h>", "<wordexp.h>",
"<langinfo.h>", "<signal.h>", "<curses.h>", "<term.h>",
"<uncntrl.h>", "<linux/"}
for _, header := range stdHeaders {
if strings.Contains(fileLine, header) {
return true
}
}
return false
}
func processSourceFiles(sourcesPath, appFolder, includeFolder string,
sourceFiles, includesFiles []string) ([]string, error) {
err := filepath.Walk(sourcesPath, func(path string, info os.FileInfo,
err error) error {
if !info.IsDir() {
extension := filepath.Ext(info.Name())
if _, ok := srcLanguages[extension]; ok {
// Add source files to sourceFiles list
@ -190,7 +456,7 @@ func processSourceFiles(sourcesPath, appFolder, includeFolder string,
if err = u.CopyFileContents(path, appFolder+info.Name()); err != nil {
return err
}
} else if extension == ".h" {
} else if extension == ".h" || extension == ".hpp" || extension == ".hcc" {
// Add source files to includesFiles list
includesFiles = append(includesFiles, info.Name())
@ -202,10 +468,8 @@ func processSourceFiles(sourcesPath, appFolder, includeFolder string,
u.PrintWarning("Unsupported extension for file: " + info.Name())
}
}
return nil
})
if err != nil {
return nil, err
}

9
srcs/common/arguments.go
View file

@ -8,9 +8,10 @@ package common
import (
"errors"
"github.com/akamensky/argparse"
"os"
"strings"
"github.com/akamensky/argparse"
)
// Exported constants to determine arguments type.
@ -18,6 +19,7 @@ const (
INT = iota
BOOL
STRING
STRINGLIST
)
// Exported constants to determine which tool is used.
@ -41,6 +43,7 @@ type Arguments struct {
IntArg map[string]*int
BoolArg map[string]*bool
StringArg map[string]*string
StringListArg map[string]*[]string
}
// InitArguments allows to initialize the parser in order to parse given
@ -52,6 +55,7 @@ func (args *Arguments) InitArguments(name, description string) (*argparse.Parser
args.IntArg = make(map[string]*int)
args.BoolArg = make(map[string]*bool)
args.StringArg = make(map[string]*string)
args.StringListArg = make(map[string]*[]string)
p := argparse.NewParser(name, description)
@ -130,5 +134,8 @@ func (*Arguments) InitArgParse(p *argparse.Parser, args *Arguments, typeVar int,
case STRING:
args.StringArg[long] = new(string)
args.StringArg[long] = p.String(short, long, options)
case STRINGLIST:
args.StringListArg[long] = new([]string)
args.StringListArg[long] = p.StringList(short, long, options)
}
}

9
srcs/common/data.go
View file

@ -6,10 +6,11 @@
package common
// Exported struct that represents static and dynamic data.
// Exported struct that represents static, dynamic and sources data.
type Data struct {
StaticData StaticData `json:"static_data"`
DynamicData DynamicData `json:"dynamic_data"`
SourcesData SourcesData `json:"sources_data"`
}
// Exported struct that represents data for static dependency analysis.
@ -26,3 +27,9 @@ type DynamicData struct {
SystemCalls map[string]int `json:"system_calls"`
Symbols map[string]string `json:"symbols"`
}
// Exported struct that represents data for sources dependency analysis.
type SourcesData struct {
SystemCalls map[string]int `json:"system_calls"`
Symbols map[string]string `json:"symbols"`
}

2
srcs/common/git.go
View file

@ -6,7 +6,7 @@
package common
const branch = "RELEASE-0.7.0"
const branch = "RELEASE-0.9.0"
// GitCloneRepository clones a git repository at the the given url.
//

6
srcs/dependtool/args.go
View file

@ -7,9 +7,10 @@
package dependtool
import (
"github.com/akamensky/argparse"
"os"
u "tools/srcs/common"
"github.com/akamensky/argparse"
)
const (
@ -52,7 +53,8 @@ func parseLocalArguments(p *argparse.Parser, args *u.Arguments) error {
Help: "Full static analysis (analyse shared libraries too)"})
args.InitArgParse(p, args, u.INT, "", typeAnalysis,
&argparse.Options{Required: false, Default: 0,
Help: "Kind of analysis (0: both; 1: static; 2: dynamic)"})
Help: "Kind of analysis (0: all; 1: static; 2: dynamic; 3: interdependence; 4: " +
"sources; 5: stripped-down app and json for buildtool)"})
return u.ParserWrapper(p, os.Args)
}

354
srcs/dependtool/interdependence_graph.go Normal file
View file

@ -0,0 +1,354 @@
package dependtool
import (
"fmt"
"io/ioutil"
"net/http"
"os"
"os/exec"
"path/filepath"
"regexp"
"strconv"
"strings"
u "tools/srcs/common"
)
// ---------------------------------Gather Data---------------------------------
// findSourceFilesAndFolders puts together all the application C/C++ source files found on one hand
// and all the application (sub-)folder paths on the other hand.
//
// It returns two slices: one containing the found source file paths and one containing the found
// (sub-)folder paths, and an error if any, otherwise it returns nil.
func findSourceFilesAndFolders(workspace string) ([]string, []string, error) {
var filenames []string
var foldernames []string
err := filepath.Walk(workspace,
func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
// Ignore hidden elements
if string(info.Name()[0]) == "." {
return nil
}
// Found (sub-)folder
if info.IsDir() {
foldernames = append(foldernames, "-I")
foldernames = append(foldernames, path)
return nil
}
// Found source file
ext := filepath.Ext(info.Name())
if ext == ".c" || ext == ".cpp" || ext == ".cc" || ext == ".h" || ext == ".hpp" ||
ext == ".hcc" {
filenames = append(filenames, path)
}
return nil
})
if err != nil {
return nil, nil, err
}
return filenames, foldernames, nil
}
// checkIncDir checks that an inclusion directive path does not contain the two dots ".." which
// refer to the parent directory and that can cause trouble when pruning the map and generating the
// graph. If the path does contain these dots, it is simplified.
//
// It returns the simplified directive path or the path itself if it contains no dots.
func checkIncDir(directive string) string {
// No problem
if !strings.Contains(directive, "../") {
return directive
}
// Must be simplified
splitDir := strings.Split(directive, u.SEP)
var i int
for i = 0; i < len(splitDir); i++ {
if splitDir[i] == ".." {
// Dots at the beginning of the path
if i == 0 {
splitDir = splitDir[1:]
i--
// Dots somewhere else in the path
} else {
splitDir = append(splitDir[:i-1], splitDir[i+1:]...)
i -= 2
}
}
}
return strings.Join(splitDir, u.SEP)
}
// sourceFileIncludesAnalysis collects all the inclusion directives from a C/C++ source file.
//
// It returns a slice containing all the relative paths contained in the inclusion directives.
func sourceFileIncludesAnalysis(sourceFile string) []string {
var fileIncDir []string
fileLines, err := u.ReadLinesFile(sourceFile)
if err != nil {
u.PrintErr(err)
}
// Find inclusion directives using regexp
var re = regexp.MustCompile(`(.*)(#include)(.*)("|<)(.*)("|>)(.*)`)
for lineIndex := range fileLines {
for _, match := range re.FindAllStringSubmatch(fileLines[lineIndex], -1) {
// Append the relative path to the list of relative paths
for i := 1; i < len(match); i++ {
if match[i] == "\"" || match[i] == "<" {
fileIncDir = append(fileIncDir, checkIncDir(match[i+1]))
break
}
}
}
}
return fileIncDir
}
// TODO REPLACE
// ExecuteCommand a single command without displaying the output.
//
// It returns a string which represents stdout and an error if any, otherwise
// it returns nil.
func executeCommand(command string, arguments []string) (string, error) {
out, err := exec.Command(command, arguments...).CombinedOutput()
return string(out), err
}
// gppSourceFileIncludesAnalysis collects all the inclusion directives from a C/C++ source file
// using the gpp preprocessor.
//
// It returns a slice containing all the absolute paths contained in the inclusion directives.
func gppSourceFileIncludesAnalysis(sourceFile, programPath string,
sourceFolders []string) ([]string, error) {
var fileIncDir []string
// g++ command
outputStr, err := executeCommand("g++", append([]string{"-E", sourceFile}, sourceFolders...))
// If command g++ returns an error, prune file: it contains non-standard libraries
if err != nil {
return nil, err
}
// Find inclusion directive paths
outputSlice := strings.Split(outputStr, "\n")
for _, line := range outputSlice {
// If warnings or errors are present, ignore their paths
if strings.Contains(line, ":") {
continue
}
// Only interested in file paths not coming from the standard library
if strings.Contains(line, programPath) {
includeDirective :=
checkIncDir(line[strings.Index(line, "\"")+1 : strings.LastIndex(line, "\"")])
if !u.Contains(fileIncDir, includeDirective) {
fileIncDir = append(fileIncDir, includeDirective)
}
}
}
return fileIncDir, nil
}
// pruneRemovableFiles prunes interdependence graph elements if the latter are unused header files.
func pruneRemovableFiles(interdependMap *map[string][]string) {
for file := range *interdependMap {
// No removal of C/C++ source files
if filepath.Ext(file) != ".c" && filepath.Ext(file) != ".cpp" &&
filepath.Ext(file) != ".cc" {
// Lookup for files depending on the current header file
depends := false
for _, dependencies := range *interdependMap {
for _, dependency := range dependencies {
if file == dependency {
depends = true
break
}
}
if depends {
break
}
}
// Prune header file if unused
if !depends {
delete(*interdependMap, file)
}
}
}
}
// pruneElemFiles prunes interdependence graph elements if the latter contain the substring in
// argument.
func pruneElemFiles(interdependMap *map[string][]string, pruneElem string) {
// Lookup for key elements containing the substring and prune them
for key := range *interdependMap {
if strings.Contains(key, pruneElem) {
delete(*interdependMap, key)
// Lookup for key elements that depend on the key found above and prune them
for file, dependencies := range *interdependMap {
for _, dependency := range dependencies {
if dependency == key {
pruneElemFiles(interdependMap, file)
}
}
}
}
}
}
// requestUnikraftExtLibs collects all the GitHub repositories of Unikraft through the GitHub API
// and returns the whole list of Unikraft external libraries.
//
// It returns a slice containing all the libraries names.
func requestUnikraftExtLibs() []string {
var extLibsList, appsList []string
// Only 2 Web pages of repositories as for february 2023 (125 repos - 100 repos per page)
nbPages := 2
for i := 1; i <= nbPages; i++ {
// HTTP Get request
resp, err := http.Get("https://api.github.com/orgs/unikraft/repos?page=" +
strconv.Itoa(i) + "&per_page=100")
if err != nil {
u.PrintErr(err)
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
u.PrintErr(err)
}
// Collect libraries
fileLines := strings.Split(string(body), "\"name\":\"lib-")
for i := 1; i < len(fileLines); i++ {
extLibsList = append(extLibsList, fileLines[i][0:strings.Index(fileLines[i][0:],
"\"")])
}
// Collect applications
fileLines = strings.Split(string(body), "\"name\":\"app-")
for i := 1; i < len(fileLines); i++ {
appsList = append(appsList, fileLines[i][0:strings.Index(fileLines[i][0:],
"\"")])
}
}
// Avoid libs that are also apps (e.g. nginx, redis)
for i, lib := range extLibsList {
if u.Contains(appsList, lib) {
extLibsList = append(extLibsList[:i], extLibsList[i+1:]...)
}
}
return extLibsList
}
// -------------------------------------Run-------------------------------------
// runInterdependAnalyser collects all the inclusion directives (i.e., dependencies) from each
// C/C++ application source file and builds an interdependence graph (dot file and png image)
// between the source files that it deemed compilable with Unikraft.
//
// It returns the path to the folder containing the source files that have been kept for generating
// the graph.
func interdependAnalyser(programPath, programName, outFolder string) string {
// Find all application source files and (sub-)folders
sourceFiles, sourceFolders, err := findSourceFilesAndFolders(programPath)
if err != nil {
u.PrintErr(err)
}
u.PrintOk(fmt.Sprint(len(sourceFiles)) + " source files found")
u.PrintOk(fmt.Sprint(len(sourceFolders)) + " source subfolders found")
// Collect source file inclusion directives. Source files are first analysed by g++ to make
// sure to avoid directives that are commented or subjected to a macro and then "by hand" to
// sort the include directives of the g++ analysis (i.e. to avoid inclusion directives that are
// not present in the source file currently analysed).
interdependMap := make(map[string][]string)
for _, sourceFile := range sourceFiles {
analysis := sourceFileIncludesAnalysis(sourceFile)
gppAnalysis, err := gppSourceFileIncludesAnalysis(sourceFile, programPath, sourceFolders)
if err != nil {
continue
}
// Build the interdependence map with the paths contained in the inclusion directives
interdependMap[sourceFile] = make([]string, 0)
for _, directive := range analysis {
for _, gppDirective := range gppAnalysis {
if strings.Contains(gppDirective, directive) {
interdependMap[sourceFile] = append(interdependMap[sourceFile], gppDirective)
}
}
}
}
// Prune the interdependence graph
extLibsList := requestUnikraftExtLibs()
extLibsList = append(extLibsList, "win32", "test", "TEST")
for _, extLib := range extLibsList {
pruneElemFiles(&interdependMap, extLib)
}
pruneRemovableFiles(&interdependMap)
// Create a folder and copy all the kept source files into it for later use with build tool
outAppFolder := outFolder + programName + u.SEP
_, err = u.CreateFolder(outAppFolder)
if err != nil {
u.PrintErr(err)
}
for _, sourceFile := range sourceFiles {
if _, ok := interdependMap[sourceFile]; ok {
if err := u.CopyFileContents(sourceFile,
outAppFolder+filepath.Base(sourceFile)); err != nil {
u.PrintErr(err)
}
}
}
u.PrintOk(fmt.Sprint(len(interdependMap)) + " source files kept and copied to " + outAppFolder)
// Change the absolute paths in the interdependence map into relative paths for more
// readability in the png image
graphMap := make(map[string][]string)
for appFilePath := range interdependMap {
appFile := strings.Split(appFilePath, programPath)[1][1:]
graphMap[appFile] = make([]string, 0)
for _, fileDepPath := range interdependMap[appFilePath] {
graphMap[appFile] = append(graphMap[appFile], strings.Split(fileDepPath,
programPath)[1][1:])
}
}
// Create dot and png files
u.GenerateGraph(programName, outFolder+programName, graphMap, nil)
return outAppFolder
}

244
srcs/dependtool/parserClang.py Executable file
View file

@ -0,0 +1,244 @@
#!/usr/bin/env python3
#---------------------------------------------------------------------
# (*) Installation:
#
# pip3 install clang
#
# cd /usr/lib/x86_64-linux-gnu/
# sudo ln -s libclang-X.Y.so.1 libclang-14.so (X.Y the version number)
#
# (*) Run:
#
# python3 parserClang.py <filepath> [includepathsfile]
#
# where filepath can be a repository/folder or a file (c/cpp/h/hpp)
#
#
# Gaulthier Gain <gaulthier.gain@uliege.be>
# License: BSD
#---------------------------------------------------------------------
import getopt
import os
import sys
import json
import clang.cindex
import clang
import platform
from clang.cindex import CursorKind
from collections import Counter
verbose = False # Change it to verbose mode
global_funcs = Counter()
global_calls = Counter()
silent_flag = False
# Check if a path is a directory or a file
def check_input_path(path, includePaths):
if os.path.isdir(path):
iterate_root_folder(path, includePaths)
elif os.path.isfile(path):
check_type_file(path, includePaths)
else:
print("Unable to analyse this file")
exit(1)
def get_include_paths(rootdir, includepathsFile):
paths = []
with open(includepathsFile, 'r') as file:
for includePath in file.readlines():
path = '-isystem ' + rootdir + includePath.replace('\n', '')
paths.append(path)
return ' '.join(paths)
# Check type/exenstion of a given file
def check_type_file(filepath, includePaths):
cplusplusOptions = '-x c++ --std=c++11'
cOptions = ''
if includePaths is not None:
cplusplusOptions = cplusplusOptions + ' ' + includePaths
cOptions = cOptions + ' ' + includePaths
if silent_flag is False:
print("Gathering symbols of " + filepath)
if filepath.endswith(".cpp") or filepath.endswith(".hpp") or filepath.endswith(".cc") or \
filepath.endswith(".hcc"):
parse_file(filepath, cplusplusOptions)
elif filepath.endswith(".c") or filepath.endswith(".h"):
parse_file(filepath, cOptions)
# Iterate through a root folder
def iterate_root_folder(rootdir, includePaths):
for subdir, dirs, files in os.walk(rootdir):
for file in files:
filepath = subdir + os.sep + file
check_type_file(filepath, includePaths)
# Print info about symbols (verbose mode)
def display_info_function(funcs, calls):
for f in funcs:
print(fully_qualified(f), f.location)
for c in calls:
if is_function_call(f, c):
print('-', c.location)
print()
# Parse a given file to generate a AST
def parse_file(filepath, arguments):
idx = clang.cindex.Index.create()
args = arguments.split()
tu = idx.parse(filepath, args=args)
funcs, calls = find_funcs_and_calls(tu)
if verbose:
display_info_function(funcs, calls)
print(list(tu.diagnostics))
# Retrieve a fully qualified function name (with namespaces)
def fully_qualified(c):
if c is None:
return ''
elif c.kind == CursorKind.TRANSLATION_UNIT:
return ''
else:
res = fully_qualified(c.semantic_parent)
if res != '':
return res + '::' + c.spelling
return c.spelling
# Determine where a call-expression cursor refers to a particular
# function declaration
def is_function_call(funcdecl, c):
defn = c.get_definition()
return (defn is not None) and (defn == funcdecl)
# Filter name to take only the function name (remove "(args)")
def filter_func_name(displayname):
if "(" in displayname:
funcName = displayname.split('(')[0]
else:
funcName = displayname
return funcName
# Retrieve lists of function declarations and call expressions in a
#translation unit
def find_funcs_and_calls(tu):
filename = tu.cursor.spelling
calls = []
funcs = []
for c in tu.cursor.walk_preorder():
if c.kind == CursorKind.CALL_EXPR:
calls.append(c)
# filter name to take only the name if necessary
funcName = filter_func_name(c.displayname)
global_calls[funcName] += 1
elif c.kind == CursorKind.FUNCTION_DECL:
funcs.append(c)
# filter name to take only the name if necessary
funcName = filter_func_name(c.displayname)
global_funcs[funcName] += 1
return funcs, calls
# Write data to json file
def write_to_json(output_filename, data):
with open(output_filename + '.json', 'w') as fp:
json.dump(data, fp, indent=4, sort_keys=True)
# Open data to json file
def read_from_json(filename):
with open(output_filename + '.json', 'r') as fp:
data = json.load(fp)
return data
# Read the list of syscalls (text file)
def read_syscalls_list(filename):
syscalls = set()
with open(filename) as f:
for line in f:
syscalls.add(line.strip())
return syscalls
# Check which syscall is called
def compare_syscalls(syscalls):
if silent_flag is False:
print("Gathered syscalls from function calls:")
return [key for key in global_calls.keys() if key not in syscalls]
# Main function
def main():
optlist, args = getopt.getopt(sys.argv[1:], "o:qvt")
input_file_names = None
includepathsFile = None
output_file_name = None
textFormat = False
for opt in optlist:
if opt[0] == "-i":
includepathFile = opt[1]
if opt[0] == "-o":
output_file_name = opt[1]
if opt[0] == "-q":
global silent_flag
silent_flag = True
if opt[0] == "-v":
global verbose
verbose = True
if opt[0] == "-t":
textFormat = True
input_file_names = args
if len(input_file_names) == 0:
if silent_flag is False:
print("No input files supplied")
exit(1)
if includepathsFile is not None:
includePaths = get_include_paths(input_file_name, includepathsFile)
for input_file_name in input_file_names:
check_input_path(input_file_name, includePaths)
else:
for input_file_name in input_file_names:
check_input_path(input_file_name, None)
if silent_flag is False:
print("---------------------------------------------------------")
if textFormat:
i = 0
for key,value in global_funcs.items():
if i < len(global_funcs.items())-1:
print(key, end=',')
else:
print(key)
i = i + 1
else:
# Dump function declarations and calls to json
output_dikt = {
'functions':'',
'calls':''
}
output_dikt['functions'] = [{'name':key, 'value':value} for key,value in
global_funcs.items()]
output_dikt['calls'] = [{'name':key, 'value':value} for key,value in global_calls.items()]
if includepathsFile is not None:
# Read syscalls from txt file
all_syscalls = read_syscalls_list('syscall_list.txt')
called_syscalls = compare_syscalls(all_syscalls)
output_dikt['syscalls'] = called_syscalls
output_file = sys.stdout
json.dump(output_dikt, output_file)
if __name__== "__main__":
if platform.system() == "Darwin":
clang.cindex.Config.set_library_file
("/Applications/Xcode.app/Contents/Frameworks/libclang.dylib")
main()

68
srcs/dependtool/run_deptool.go
View file

@ -4,12 +4,32 @@ import (
"debug/elf"
"errors"
"fmt"
"github.com/fatih/color"
"runtime"
"strings"
u "tools/srcs/common"
"github.com/fatih/color"
)
// getProgramFolder gets the folder path in which the given program is located, according to the
// Unikraft standard (e.g., /home/.../apps/programFolder/.../program).
//
// It returns the folder containing the program files according to the standard described above.
func getProgramFolder(programPath string) string {
tmp := strings.Split(programPath, u.SEP)
i := 2
for ; i < len(tmp); i++ {
if tmp[len(tmp)-i] == "apps" {
break
}
}
folderPath := strings.Join(tmp[:len(tmp)-i+2], u.SEP)
return folderPath
}
// RunAnalyserTool allows to run the dependency analyser tool.
func RunAnalyserTool(homeDir string, data *u.Data) {
@ -29,10 +49,11 @@ func RunAnalyserTool(homeDir string, data *u.Data) {
u.PrintErr(err)
}
// Get the kind of analysis (0: both; 1: static; 2: dynamic)
// Get the kind of analysis (0: all; 1: static; 2: dynamic; 3: interdependence; 4: sources; 5:
// stripped-down app and json for buildtool)
typeAnalysis := *args.IntArg[typeAnalysis]
if typeAnalysis < 0 || typeAnalysis > 2 {
u.PrintErr(errors.New("analysis argument must be between [0,2]"))
if typeAnalysis < 0 || typeAnalysis > 5 {
u.PrintErr(errors.New("analysis argument must be between [0,5]"))
}
// Get program path
@ -68,7 +89,8 @@ func RunAnalyserTool(homeDir string, data *u.Data) {
// Run static analyser
if typeAnalysis == 0 || typeAnalysis == 1 {
u.PrintHeader1("(1.1) RUN STATIC ANALYSIS")
runStaticAnalyser(elfFile, isDynamic, isLinux, args, programName, programPath, outFolder, data)
runStaticAnalyser(elfFile, isDynamic, isLinux, args, programName, programPath, outFolder,
data)
}
// Run dynamic analyser
@ -82,6 +104,24 @@ func RunAnalyserTool(homeDir string, data *u.Data) {
}
}
// Run interdependence analyser
if typeAnalysis == 0 || typeAnalysis == 3 {
u.PrintHeader1("(1.3) RUN INTERDEPENDENCE ANALYSIS")
_ = runInterdependAnalyser(getProgramFolder(programPath), programName, outFolder)
}
// Run sources analyser
if typeAnalysis == 0 || typeAnalysis == 4 {
u.PrintHeader1("(1.4) RUN SOURCES ANALYSIS")
runSourcesAnalyser(getProgramFolder(programPath), data)
}
// Prepare stripped-down app for buildtool
if typeAnalysis == 5 {
u.PrintHeader1("(1.5) PREPARE STRIPPED-DOWN APP AND JSON FOR BUILDTOOL")
runSourcesAnalyser(runInterdependAnalyser(programPath, programName, outFolder), data)
}
// Save Data to JSON
if err = u.RecordDataJson(outFolder+programName, data); err != nil {
u.PrintErr(err)
@ -156,8 +196,8 @@ func checkElf(programPath *string) (*elf.File, bool) {
}
// runStaticAnalyser runs the static analyser
func runStaticAnalyser(elfFile *elf.File, isDynamic, isLinux bool, args *u.Arguments, programName, programPath,
outFolder string, data *u.Data) {
func runStaticAnalyser(elfFile *elf.File, isDynamic, isLinux bool, args *u.Arguments, programName,
programPath, outFolder string, data *u.Data) {
staticAnalyser(elfFile, isDynamic, isLinux, *args, data, programPath)
@ -209,6 +249,12 @@ func runDynamicAnalyser(args *u.Arguments, programName, programPath,
}
}
// runDynamicAnalyser runs the sources analyser.
func runSourcesAnalyser(programPath string, data *u.Data) {
sourcesAnalyser(data, programPath)
}
// saveGraph saves dependency graphs of a given app into the output folder.
func saveGraph(programName, outFolder string, data *u.Data) {
@ -222,8 +268,14 @@ func saveGraph(programName, outFolder string, data *u.Data) {
programName+"_dependencies", data.StaticData.Dependencies, nil)
}
if len(data.StaticData.SharedLibs) > 0 {
if len(data.DynamicData.SharedLibs) > 0 {
u.GenerateGraph(programName, outFolder+"dynamic"+u.SEP+
programName+"_shared_libs", data.DynamicData.SharedLibs, nil)
}
}
// runInterdependAnalyser runs the interdependence analyser.
func runInterdependAnalyser(programPath, programName, outFolder string) string {
return interdependAnalyser(programPath, programName, outFolder)
}

113
srcs/dependtool/sources_analyser.go Normal file
View file

@ -0,0 +1,113 @@
package dependtool
import (
"fmt"
"os"
"path/filepath"
"strings"
u "tools/srcs/common"
)
// ---------------------------------Gather Data---------------------------------
// findSourcesFiles puts together all C/C++ source files found in a given application folder.
//
// It returns a slice containing the found source file names and an error if any, otherwise it
// returns nil.
func findSourceFiles(workspace string) ([]string, error) {
var filenames []string
err := filepath.Walk(workspace,
func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
ext := filepath.Ext(info.Name())
if ext == ".c" || ext == ".cpp" || ext == ".cc" || ext == ".h" || ext == ".hpp" ||
ext == ".hcc" {
filenames = append(filenames, path)
}
return nil
})
if err != nil {
return nil, err
}
return filenames, nil
}
// addSourceFileSymbols adds all the symbols present in 'output' to the static data field in
// 'data'.
func addSourceFileSymbols(output string, data *u.SourcesData) {
outputTab := strings.Split(output, ",")
// Get the list of system calls
systemCalls := initSystemCalls()
for _, s := range outputTab {
if _, isSyscall := systemCalls[s]; isSyscall {
data.SystemCalls[s] = systemCalls[s]
} else {
data.Symbols[s] = ""
}
}
}
// extractPrototype executes the parserClang.py script on each source file to extracts all possible
// symbols of each of these files.
//
// It returns an error if any, otherwise it returns nil.
func extractPrototype(sourcesFiltered []string, data *u.SourcesData) error {
for _, f := range sourcesFiltered {
script := filepath.Join(os.Getenv("GOPATH"), "src", "tools", "srcs", "dependtool",
"parserClang.py")
output, err := u.ExecuteCommand("python3", []string{script, "-q", "-t", f})
if err != nil {
u.PrintWarning("Incomplete analysis with file " + f)
continue
}
addSourceFileSymbols(output, data)
}
return nil
}
// gatherSourceFileSymbols gathers symbols of source files from a given application folder.
//
// It returns an error if any, otherwise it returns nil.
func gatherSourceFileSymbols(data *u.SourcesData, programPath string) error {
sourceFiles, err := findSourceFiles(programPath)
if err != nil {
u.PrintErr(err)
}
u.PrintOk(fmt.Sprint(len(sourceFiles)) + " source files found")
if err := extractPrototype(sourceFiles, data); err != nil {
u.PrintErr(err)
}
return nil
}
// -------------------------------------Run-------------------------------------
// staticAnalyser runs the static analysis to get system calls and library calls of a given
// application.
func sourcesAnalyser(data *u.Data, programPath string) {
sourcesData := &data.SourcesData
// Init symbols members
sourcesData.Symbols = make(map[string]string)
sourcesData.SystemCalls = make(map[string]int)
// Detect symbols from source files
u.PrintHeader2("(*) Gathering symbols from source files")
if err := gatherSourceFileSymbols(sourcesData, programPath); err != nil {
u.PrintWarning(err)
}
}

7
srcs/dependtool/static_analyser.go
View file

@ -195,8 +195,8 @@ func executeDependAptCache(programName string, data *u.StaticData,
// staticAnalyser runs the static analysis to get shared libraries,
// system calls and library calls of a given application.
//
func staticAnalyser(elfFile *elf.File, isDynamic, isLinux bool, args u.Arguments, data *u.Data, programPath string) {
func staticAnalyser(elfFile *elf.File, isDynamic, isLinux bool, args u.Arguments, data *u.Data,
programPath string) {
programName := *args.StringArg[programArg]
fullDeps := *args.BoolArg[fullDepsArg]
@ -242,7 +242,8 @@ func staticAnalyser(elfFile *elf.File, isDynamic, isLinux bool, args u.Arguments
// Detect symbols from shared libraries
if fullStaticAnalysis && isLinux {
u.PrintHeader2("(*) Gathering symbols and system calls of shared libraries from binary file")
u.PrintHeader2("(*) Gathering symbols and system calls of shared libraries from binary" +
"file")
for key, path := range staticData.SharedLibs {
if len(path) > 0 {
fmt.Printf("\t-> Analysing %s - %s\n", key, path[0])

5
srcs/extractertool/parserClang.py
View file

@ -5,7 +5,7 @@
# pip3 install clang
#
# cd /usr/lib/x86_64-linux-gnu/
# sudo ln -s libclang-X.Y.so.1 libclang.so (X.Y the version number)
# sudo ln -s libclang-X.Y.so.1 libclang-14.so (X.Y the version number)
#
# (*) Run:
#
@ -64,7 +64,8 @@ def check_type_file(filepath, includePaths):
cOptions = cOptions + ' ' + includePaths
if silent_flag is False:
print("Gathering symbols of " + filepath)
if filepath.endswith(".cpp") or filepath.endswith(".hpp"):
if filepath.endswith(".cpp") or filepath.endswith(".hpp") or filepath.endswith(".cc") or \
filepath.endswith(".hcc"):
parse_file(filepath, cplusplusOptions)
elif filepath.endswith(".c") or filepath.endswith(".h"):
parse_file(filepath, cOptions)

118
srcs/extractertool/run_extractertool.go
View file

@ -6,7 +6,6 @@ import (
"io/ioutil"
"net/http"
"os"
"os/exec"
"path/filepath"
"regexp"
"strconv"
@ -32,7 +31,7 @@ type Variables struct {
}
func getMakefileSources(content string, mapSources map[string]string) {
var re = regexp.MustCompile(`(?m)\/.*\.c|\/.*\.h|\/.*\.cpp`)
var re = regexp.MustCompile(`(?m)\/.*\.c|\/.*\.h|\/.*\.cpp|\/.*\.hpp|\/.*\.cc|\/.*\.hcc`)
for _, match := range re.FindAllString(content, -1) {
vars := strings.Split(match, "/")
mapSources[vars[len(vars)-1]] = match
@ -51,7 +50,7 @@ func findVariables(content string, mapVariables map[string]*Variables) {
value: "",
}
regexVar := regexp.MustCompile("(?m)" + v.name + "=.*$")
regexVar := regexp.MustCompile("(?m)" + v.name + "[ \t]*=.*$")
for _, matchVar := range regexVar.FindAllString(content, -1) {
v.value = matchVar
break
@ -93,7 +92,8 @@ func resolveVariables(mapVariables map[string]*Variables) {
func detectURL(mapVariables map[string]*Variables) *string {
for key, value := range mapVariables {
if strings.Contains(key, URL) && strings.Contains(value.value, HTTP) {
vars := strings.Split(value.value, "=")
spaceDel := strings.Join(strings.Split(value.value, " "), "")
vars := strings.Split(spaceDel, "=")
if len(vars) > 1 {
return &vars[1]
}
@ -103,7 +103,7 @@ func detectURL(mapVariables map[string]*Variables) *string {
return nil
}
//TODO REPLACE
// TODO REPLACE
func CreateFolder(path string) (bool, error) {
if _, err := os.Stat(path); os.IsNotExist(err) {
if err = os.Mkdir(path, 0755); err != nil {
@ -146,7 +146,8 @@ func findSourcesFiles(workspace string) ([]string, error) {
}
ext := filepath.Ext(info.Name())
if ext == ".c" || ext == ".cpp" {
if ext == ".c" || ext == ".cpp" || ext == ".cc" || ext == ".h" || ext == ".hpp" ||
ext == ".hcc" {
filenames = append(filenames, path)
}
return nil
@ -157,19 +158,6 @@ func findSourcesFiles(workspace string) ([]string, error) {
return filenames, nil
}
//TODO REPLACE
// ExecuteCommand a single command without displaying the output.
//
// It returns a string which represents stdout and an error if any, otherwise
// it returns nil.
func ExecuteCommand(command string, arguments []string) (string, error) {
out, err := exec.Command(command, arguments...).CombinedOutput()
if err != nil {
return "", err
}
return string(out), nil
}
func saveSymbols(output string, mapSymbols map[string]string, libName string) {
if strings.Contains(output, "\n") {
output = strings.TrimSuffix(output, "\n")
@ -179,7 +167,7 @@ func saveSymbols(output string, mapSymbols map[string]string, libName string) {
for _, s := range symbols {
if len(s) > 0 {
if _, ok := mapSymbols[s]; !ok {
if s == "main" || strings.Contains(s, "test") {
if s == "main" || strings.Contains(s, "test") || strings.Contains(s, "TEST") {
u.PrintWarning("Ignore function: " + s)
} else {
mapSymbols[s] = libName
@ -189,11 +177,13 @@ func saveSymbols(output string, mapSymbols map[string]string, libName string) {
}
}
func extractPrototype(sourcesFiltered []string, mapSymbols map[string]string, libName string) error {
func extractPrototype(sourcesFiltered []string, mapSymbols map[string]string,
libName string) error {
for _, f := range sourcesFiltered {
script := filepath.Join(os.Getenv("GOPATH"), "src", "tools", "srcs", "extractertool", "parserClang.py")
output, err := ExecuteCommand("python3", []string{script, "-q", "-t", f})
script := filepath.Join(os.Getenv("GOPATH"), "src", "tools", "srcs", "extractertool",
"parserClang.py")
output, err := u.ExecuteCommand("python3", []string{script, "-q", "-t", f})
if err != nil {
u.PrintWarning("Incomplete analysis with file " + f)
continue
@ -222,7 +212,8 @@ func RunExtracterTool(homeDir string) {
// Init and parse local arguments
args := new(u.Arguments)
p, err := args.InitArguments("--extracter",
"The extracter tool allows to extract all the symbols (functions) of an external/internal library")
"The extracter tool allows to extract all the symbols (functions) of an "+
"external/internal library")
if err != nil {
u.PrintErr(err)
}
@ -252,28 +243,41 @@ func RunExtracterTool(homeDir string) {
mapSources := make(map[string]string)
getMakefileSources(content, mapSources)
findVariables(content, mapVariables)
resolveVariables(mapVariables)
url := detectURL(mapVariables)
if url == nil {
u.PrintErr(errors.New("url of the lib not found"))
return
folderName := lib + "_sources_folder"
var archiveName string
var sourcesFiltered []string
if url != nil {
var fileExtension string
urlSplit := strings.Split(*url, "/")
if urlSplit[len(urlSplit)-1] == "download" {
fileExtension = filepath.Ext(urlSplit[len(urlSplit)-2])
} else {
fileExtension = filepath.Ext(*url)
}
fileExtension := filepath.Ext(*url)
folderName := lib + "_sources_folder"
created, err := CreateFolder(folderName)
if err != nil {
u.PrintErr(err)
}
var files []string
archiveName := lib + "_sources" + fileExtension
if fileExtension == ".gz" || fileExtension == ".xz" {
archiveName = lib + "_sources.tar" + fileExtension
} else {
archiveName = lib + "_sources" + fileExtension
}
if created {
u.PrintInfo(*url + " is found. Download the lib sources...")
err := DownloadFile(archiveName, *url)
if err != nil {
_ = os.RemoveAll(folderName)
u.PrintErr(err)
}
u.PrintOk(*url + " successfully downloaded.")
@ -287,7 +291,8 @@ func RunExtracterTool(homeDir string) {
u.PrintErr(err.Error() + ". Corrupted archive. Please try again.")
}
} else if fileExtension == ".tar" || fileExtension == ".gz" || fileExtension == ".tgz" {
} else if fileExtension == ".tar" || fileExtension == ".gz" || fileExtension ==
".tgz" {
files, err = unTarGz(archiveName, folderName)
if err != nil {
_ = os.Remove(archiveName)
@ -295,26 +300,53 @@ func RunExtracterTool(homeDir string) {
u.PrintErr(err.Error() + ". Corrupted archive. Please try again.")
}
} else if fileExtension == ".tar" || fileExtension == ".xz" || fileExtension ==
".txz" {
_, err := u.ExecuteCommand("tar", []string{"-xf", archiveName, "-C", folderName})
if err != nil {
_ = os.Remove(archiveName)
_ = os.RemoveAll(folderName)
u.PrintErr(err.Error() + ". Corrupted archive. Please try again.")
}
} else {
_ = os.Remove(archiveName)
_ = os.RemoveAll(folderName)
u.PrintErr(errors.New("unknown extension for archive"))
}
}
if len(files) == 0 {
u.PrintInfo("Inspecting folder " + folderName + " for sources...")
files, err = findSourcesFiles(folderName)
folderFiles, err := findSourcesFiles(folderName)
if err != nil {
_ = os.Remove(archiveName)
_ = os.RemoveAll(folderName)
u.PrintErr(err)
}
sourcesFiltered = filterSourcesFiles(files, mapSources)
sourcesFiltered = append(sourcesFiltered, folderFiles...)
}
sourcesFiltered := filterSourcesFiles(files, mapSources)
libpathFiles, err := findSourcesFiles(libpath)
if err != nil {
if url != nil {
_ = os.Remove(archiveName)
_ = os.RemoveAll(folderName)
}
u.PrintErr(err)
}
sourcesFiltered = append(sourcesFiltered, libpathFiles...)
u.PrintInfo("Find " + strconv.Itoa(len(sourcesFiltered)) + " files to analyse")
mapSymbols := make(map[string]string)
u.PrintInfo("Extracting symbols from all sources of " + lib + ". This may take some times...")
if err := extractPrototype(sourcesFiltered, mapSymbols, lib); err != nil {
if url != nil {
_ = os.Remove(archiveName)
_ = os.RemoveAll(folderName)
}
u.PrintErr(err)
}
@ -328,14 +360,20 @@ func RunExtracterTool(homeDir string) {
u.PrintOk(strconv.Itoa(len(mapSymbols)) + " symbols from " + lib + " have been extracted.")
filename := filepath.Join(os.Getenv("GOPATH"), "src", "tools", "libs", "external", lib)
var filename string
if url != nil {
u.PrintInfo("Remove folders " + archiveName + " and " + folderName)
_ = os.Remove(archiveName)
_ = os.RemoveAll(folderName)
filename = filepath.Join(os.Getenv("GOPATH"), "src", "tools", "libs", "external", lib)
} else {
filename = filepath.Join(os.Getenv("GOPATH"), "src", "tools", "libs", "internal", lib)
}
if err := u.RecordDataJson(filename, mf); err != nil {
u.PrintErr(err)
} else {
u.PrintOk("Symbols file have been written to " + filename + ".json")
}
u.PrintInfo("Remove folders " + archiveName + " and " + folderName)
_ = os.Remove(archiveName)
_ = os.RemoveAll(folderName)
}