diff --git a/debian/changelog b/debian/changelog index 4f27c54..eb7f0d7 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,11 @@ +osrframework (0.18.0-0kali1) kali-dev; urgency=medium + + * Import new upstream release (Closes 4469) + * Update patches and installation: upstream created separated projects for + osrframework_server, osrframework_maltego and osrframework_console. + + -- Sophie Brun Wed, 24 Jan 2018 09:35:54 +0100 + osrframework (0.17.2-0kali1) kali-dev; urgency=medium * Initial release (Closes: 3042) diff --git a/debian/control b/debian/control index c8e0b93..f4b4914 100644 --- a/debian/control +++ b/debian/control @@ -7,7 +7,6 @@ Homepage: https://github.com/i3visio/osrframework Vcs-Git: git://git.kali.org/packages/osrframework.git Vcs-Browser: http://git.kali.org/gitweb/?p=packages/osrframework.git;a=summary -Testsuite: autopkgtest-pkg-python Package: osrframework Architecture: all @@ -16,7 +15,4 @@ This package contains a set of libraries developed by i3visio to perform Open Source Intelligence tasks. They include references to a bunch of different applications related to username checking, DNS lookups, information leaks - research, deep web search, regular expressions extraction and many others. At - the same time, by means of ad-hoc Maltego transforms, OSRFramework provides a - way of making these queries graphically as well as several interfaces to - interact with like OSRFConsole or a Web interface. + research, deep web search, regular expressions extraction and many others. diff --git a/debian/copyright b/debian/copyright index 5c410f9..0bc7303 100644 --- a/debian/copyright +++ b/debian/copyright @@ -3,11 +3,11 @@ Source: https://github.com/i3visio/osrframework Files: * -Copyright: 2014-2017 F. Brezo and Y. Rubio, i3visio +Copyright: 2014-2018 F. Brezo and Y. Rubio, i3visio License: AGPL-3+ Files: debian/* -Copyright: 2017 Sophie Brun +Copyright: 2017-2018 Sophie Brun License: AGPL-3+ License: AGPL-3+ diff --git a/debian/osrframework.docs b/debian/osrframework.docs index 176d429..968f11e 100644 --- a/debian/osrframework.docs +++ b/debian/osrframework.docs @@ -1,3 +1,2 @@ -osrframework/transforms/lib/TUTORIAL-MALTEGO.txt README.md doc/* diff --git a/debian/osrframework.install b/debian/osrframework.install deleted file mode 100644 index 393af6d..0000000 --- a/debian/osrframework.install +++ /dev/null @@ -1,2 +0,0 @@ -osrframework/static osrframework/templates /etc/osrframework/server -osrframework/transforms/lib/osrframework-maltego-settings /etc/osrframework/default diff --git a/debian/patches/add-script-config-upgrade.patch b/debian/patches/add-script-config-upgrade.patch index 0dd614d..8dce3b8 100644 --- a/debian/patches/add-script-config-upgrade.patch +++ b/debian/patches/add-script-config-upgrade.patch @@ -1,3 +1,16 @@ +Description: Add a script to manage configuration during upgrade + During a upgrade we have to install the new configuration files in + ~/.config/OSRFramework: + - Update configuration.py to create the missing directory "data" + only if it doesn't exist. + - Add a script (which is called in postinst when it's an upgrade) to + remove config directories "default", "plugins", "server", "transforms", + and install the new directories and files provided by the package. +Author: Sophie Brun +Origin: vendor +Last-Update: 2018-01-24 +--- +This patch header follows DEP-3: http://dep.debian.net/deps/dep3/ --- /dev/null +++ b/osrframework/utils/upgrade_config.py @@ -0,0 +1,12 @@ @@ -15,8 +28,8 @@ + configuration.initialize_config_files() --- a/osrframework/utils/configuration.py +++ b/osrframework/utils/configuration.py -@@ -40,8 +40,10 @@ def initialize_config_files(): - for dirconf in ["default", "plugins", "server"]: +@@ -41,8 +41,10 @@ def initialize_config_files(): + for dirconf in ["default", "plugins"]: shutil.copytree(os.path.join('/etc/osrframework',dirconf), os.path.join(LOCAL_CONFIG_DIR, dirconf)) - # Create directory data @@ -27,4 +40,4 @@ + os.mkdir(getConfigPath()["appPathData"]) # Create directory Transforms before copy *.py files - os.mkdir(LOCAL_CONFIG_TRANSFORMS_DIR) + # os.mkdir(LOCAL_CONFIG_TRANSFORMS_DIR) diff --git a/debian/patches/cleanup-setup.patch b/debian/patches/cleanup-setup.patch index aa9cb76..6f9b819 100644 --- a/debian/patches/cleanup-setup.patch +++ b/debian/patches/cleanup-setup.patch @@ -61,7 +61,7 @@ - - command = "echo '''" + new_lines + "''' >> ~/.bashrc" - print "[*] As we want to be transparent, the command that is being run is the following:\n" + command -- a = os.popen(command).read() +- a = os.popen(command).read() - except: - IS_VIRTUAL_ENV = True - else: @@ -71,7 +71,7 @@ + +# command = "echo '''" + new_lines + "''' >> ~/.bashrc" +# print "[*] As we want to be transparent, the command that is being run is the following:\n" + command -+# a = os.popen(command).read() ++# a = os.popen(command).read() +# except: +# IS_VIRTUAL_ENV = True +# else: @@ -104,8 +104,6 @@ - print "[*] No OSRFramework installation found in the system." -else: - print "[*] OSRFramework seems to be installed using `virtualenv`." -- --HERE = os.path.abspath(os.path.dirname(__file__)) +# if os.path.isdir(osrframeworkSystemPath): +# print "[!] Found an old installation at: " + osrframeworkSystemPath +# try: @@ -119,7 +117,8 @@ +# print "[*] No OSRFramework installation found in the system." +#else: +# print "[*] OSRFramework seems to be installed using `virtualenv`." -+# + +-HERE = os.path.abspath(os.path.dirname(__file__)) +#HERE = os.path.abspath(os.path.dirname(__file__)) # Importing the temporal scripts for the setup and taking the new version number @@ -149,7 +148,7 @@ # Reading the .md file try: -@@ -218,101 +218,101 @@ setup( +@@ -206,49 +206,49 @@ setup( ############################ ### Creating other files ### ############################ @@ -177,52 +176,11 @@ - os.path.join("config", "browser.cfg"), - os.path.join("config", "general.cfg"), - ], -- paths["appPathTransforms"] : [ -- os.path.join("osrframework", "alias_generator.py"), -- os.path.join("osrframework", "entify.py"), -- os.path.join("osrframework", "phonefy.py"), -- os.path.join("osrframework", "searchfy.py"), -- os.path.join("osrframework", "mailfy.py"), -- os.path.join("osrframework", "usufy.py"), -- os.path.join("osrframework", "domainfy.py"), -- os.path.join("osrframework", "transforms", "aliasToKnownDomains.py"), -- os.path.join("osrframework", "transforms", "aliasToKnownEmails.py"), -- os.path.join("osrframework", "transforms", "aliasToSkypeAccounts.py"), -- os.path.join("osrframework", "transforms", "aliasToSkypeIP.py"), -- os.path.join("osrframework", "transforms", "bitcoinAddressToBlockchainDetails.py"), -- os.path.join("osrframework", "transforms", "coordinatesToGoogleMapsBrowser.py"), -- os.path.join("osrframework", "transforms", "coordinatesToTwitterBrowser.py"), -- os.path.join("osrframework", "transforms", "domainToGoogleSearchUriWithEmails.py"), -- os.path.join("osrframework", "transforms", "domainToTld.py"), -- os.path.join("osrframework", "transforms", "emailToAlias.py"), -- os.path.join("osrframework", "transforms", "emailToBreachedAccounts.py"), -- os.path.join("osrframework", "transforms", "emailToDomain.py"), -- os.path.join("osrframework", "transforms", "emailToSkypeAccounts.py"), -- os.path.join("osrframework", "transforms", "expandPropertiesFromI3visioEntity.py"), -- os.path.join("osrframework", "transforms", "hashToMD5crackDotCom.py"), -- os.path.join("osrframework", "transforms", "ipToIp_ApiInformation.py"), -- os.path.join("osrframework", "transforms", "phoneToMoreInfo.py"), -- os.path.join("osrframework", "transforms", "phoneToPerson.py"), -- os.path.join("osrframework", "transforms", "textToEntities.py"), -- os.path.join("osrframework", "transforms", "textToGoogleSearchUri.py"), -- os.path.join("osrframework", "transforms", "textToPlatformSearch.py"), -- os.path.join("osrframework", "transforms", "textToProfiles.py"), -- os.path.join("osrframework", "transforms", "uriToBrowser.py"), -- os.path.join("osrframework", "transforms", "uriToDomain.py"), -- os.path.join("osrframework", "transforms", "uriToEntities.py"), -- os.path.join("osrframework", "transforms", "uriToGoogleCacheUri.py"), -- os.path.join("osrframework", "transforms", "uriToPort.py"), -- os.path.join("osrframework", "transforms", "uriToProtocol.py"), -- ], - paths["appPathWrappers"] : [ - os.path.join("config", "plugins", "wrapper.py.sample"), - ], - paths["appPathPatterns"] : [ - os.path.join("config", "plugins", "pattern.py.sample"), -- ], -- paths["appPathServer"] : [ -- os.path.join("osrframework", "static"), -- os.path.join("osrframework", "templates"), - ] -} +#print "[*] Copying relevant files..." @@ -237,66 +195,23 @@ +# os.path.join("config", "browser.cfg"), +# os.path.join("config", "general.cfg"), +# ], -+# paths["appPathTransforms"] : [ -+# os.path.join("osrframework", "alias_generator.py"), -+# os.path.join("osrframework", "entify.py"), -+# os.path.join("osrframework", "phonefy.py"), -+# os.path.join("osrframework", "searchfy.py"), -+# os.path.join("osrframework", "mailfy.py"), -+# os.path.join("osrframework", "usufy.py"), -+# os.path.join("osrframework", "domainfy.py"), -+# os.path.join("osrframework", "transforms", "aliasToKnownDomains.py"), -+# os.path.join("osrframework", "transforms", "aliasToKnownEmails.py"), -+# os.path.join("osrframework", "transforms", "aliasToSkypeAccounts.py"), -+# os.path.join("osrframework", "transforms", "aliasToSkypeIP.py"), -+# os.path.join("osrframework", "transforms", "bitcoinAddressToBlockchainDetails.py"), -+# os.path.join("osrframework", "transforms", "coordinatesToGoogleMapsBrowser.py"), -+# os.path.join("osrframework", "transforms", "coordinatesToTwitterBrowser.py"), -+# os.path.join("osrframework", "transforms", "domainToGoogleSearchUriWithEmails.py"), -+# os.path.join("osrframework", "transforms", "domainToTld.py"), -+# os.path.join("osrframework", "transforms", "emailToAlias.py"), -+# os.path.join("osrframework", "transforms", "emailToBreachedAccounts.py"), -+# os.path.join("osrframework", "transforms", "emailToDomain.py"), -+# os.path.join("osrframework", "transforms", "emailToSkypeAccounts.py"), -+# os.path.join("osrframework", "transforms", "expandPropertiesFromI3visioEntity.py"), -+# os.path.join("osrframework", "transforms", "hashToMD5crackDotCom.py"), -+# os.path.join("osrframework", "transforms", "ipToIp_ApiInformation.py"), -+# os.path.join("osrframework", "transforms", "phoneToMoreInfo.py"), -+# os.path.join("osrframework", "transforms", "phoneToPerson.py"), -+# os.path.join("osrframework", "transforms", "textToEntities.py"), -+# os.path.join("osrframework", "transforms", "textToGoogleSearchUri.py"), -+# os.path.join("osrframework", "transforms", "textToPlatformSearch.py"), -+# os.path.join("osrframework", "transforms", "textToProfiles.py"), -+# os.path.join("osrframework", "transforms", "uriToBrowser.py"), -+# os.path.join("osrframework", "transforms", "uriToDomain.py"), -+# os.path.join("osrframework", "transforms", "uriToEntities.py"), -+# os.path.join("osrframework", "transforms", "uriToGoogleCacheUri.py"), -+# os.path.join("osrframework", "transforms", "uriToPort.py"), -+# os.path.join("osrframework", "transforms", "uriToProtocol.py"), -+# ], +# paths["appPathWrappers"] : [ +# os.path.join("config", "plugins", "wrapper.py.sample"), +# ], +# paths["appPathPatterns"] : [ +# os.path.join("config", "plugins", "pattern.py.sample"), -+# ], -+# paths["appPathServer"] : [ -+# os.path.join("osrframework", "static"), -+# os.path.join("osrframework", "templates"), +# ] +#} # Iterating through all destinations to write the info -for destiny in files_to_copy.keys(): -- # Grabbing each source file to be moved ++#for destiny in files_to_copy.keys(): + # Grabbing each source file to be moved - for sourceFile in files_to_copy[destiny]: - fileToMove = os.path.join(HERE,sourceFile) -- -+#for destiny in files_to_copy.keys(): -+# # Grabbing each source file to be moved +# for sourceFile in files_to_copy[destiny]: +# fileToMove = os.path.join(HERE,sourceFile) -+# + # Choosing the command depending on the SO - if sys.platform == 'win32': - if os.path.isdir(fileToMove): @@ -321,23 +236,3 @@ #print cmd - output = os.popen(cmd).read() +# output = os.popen(cmd).read() - --print --print "[*] Last part: trying to configure Maltego Transforms..." -+#print -+#print "[*] Last part: trying to configure Maltego Transforms..." - # Creating the configuration file --try: -- import osrframework.transforms.lib.configure_maltego as maltego -- maltego.configureMaltego(transformsConfigFolder=paths["appPathTransforms"], base=os.path.join(HERE,"osrframework/transforms/lib/osrframework-maltego-settings"), debug=False, backupPath=paths["appPathDefaults"]) --except Exception, e: -- print "[!] The Maltego configuration file to use i3visio transforms could not be created and thus, cannot be used. Check the following error:" -- print str(e) --print -+#try: -+# import osrframework.transforms.lib.configure_maltego as maltego -+# maltego.configureMaltego(transformsConfigFolder=paths["appPathTransforms"], base=os.path.join(HERE,"osrframework/transforms/lib/osrframework-maltego-settings"), debug=False, backupPath=paths["appPathDefaults"]) -+#except Exception, e: -+# print "[!] The Maltego configuration file to use i3visio transforms could not be created and thus, cannot be used. Check the following error:" -+# print str(e) -+#print diff --git a/debian/patches/dont-use-validate-email.patch b/debian/patches/dont-use-validate-email.patch index 0b887f8..242882b 100644 --- a/debian/patches/dont-use-validate-email.patch +++ b/debian/patches/dont-use-validate-email.patch @@ -8,18 +8,18 @@ This patch header follows DEP-3: http://dep.debian.net/deps/dep3/ --- a/osrframework/mailfy.py +++ b/osrframework/mailfy.py -@@ -42,7 +42,7 @@ import sys +@@ -33,7 +33,7 @@ import sys # Email verification libraries import emailahoy -import validate_email +#import validate_email + import osrframework import osrframework.thirdparties.haveibeenpwned_com.hibp as hibp - import osrframework.utils.banner as banner --- a/setup.py +++ b/setup.py -@@ -196,7 +196,7 @@ setup( +@@ -184,7 +184,7 @@ setup( "tweepy", "networkx", "decorator", diff --git a/debian/patches/drop-explicit-need-of-importlib.patch b/debian/patches/drop-explicit-need-of-importlib.patch index a63cd64..6abe1eb 100644 --- a/debian/patches/drop-explicit-need-of-importlib.patch +++ b/debian/patches/drop-explicit-need-of-importlib.patch @@ -7,7 +7,7 @@ This patch header follows DEP-3: http://dep.debian.net/deps/dep3/ --- a/setup.py +++ b/setup.py -@@ -201,7 +201,7 @@ setup( +@@ -189,7 +189,7 @@ setup( "tabulate", "oauthlib>=1.0.0", # Added to dinamically import wrappers: diff --git a/debian/patches/fix-alias-generator.patch b/debian/patches/fix-alias-generator.patch new file mode 100644 index 0000000..710ba94 --- /dev/null +++ b/debian/patches/fix-alias-generator.patch @@ -0,0 +1,51 @@ +Description: alias_generator contains two functions "main" + alias_generator.py contains two functions named "main". The command fails + with the error message + "TypeError: main() got an unexpected keyword argument 'name' " +Author: Sophie Brun +Forwarded: https://github.com/i3visio/osrframework/issues/302 +Last-Update: 2018-01-24 +--- +This patch header follows DEP-3: http://dep.debian.net/deps/dep3/ +--- a/osrframework/alias_generator.py ++++ b/osrframework/alias_generator.py +@@ -45,7 +45,7 @@ LEET_TRANSFORMS = { + } + + +-def main(name=None, surname1=None, surname2=None, city=None, country=None, year=None, useNumbers=False, useCommonWords=False, useLeet=False, useLocales=False, extraWords=[]): ++def mainalias(name=None, surname1=None, surname2=None, city=None, country=None, year=None, useNumbers=False, useCommonWords=False, useLeet=False, useLocales=False, extraWords=[], outputFile="./output.txt"): + """ + The main method that generates the given aliases. + +@@ -2808,9 +2808,9 @@ def main(name=None, surname1=None, surna + print("\nGenerated nicks:\n") + print(general.success(json.dumps(listaFinal, indent=2, sort_keys=True))) + print("\nUp to " + general.emphasis(str(len(listaFinal))) + " nicks generated.\n") +- print("Writing the results onto the file:\n\t" + general.emphasis(args.outputFile)) ++ print("Writing the results onto the file:\n\t" + general.emphasis(outputFile)) + +- oF=open(args.outputFile, "w") ++ oF=open(outputFile, "w") + for l in listaFinal: + oF.write(l+"\n") + oF.close() +@@ -2901,7 +2901,7 @@ def main(params=None): + if args.country != "": + print("Country: ".ljust(20, " ") + args.country) + +- main( ++ mainalias( + name=args.name, + surname1=args.surname1, + surname2=args.surname2, +@@ -2912,7 +2912,8 @@ def main(params=None): + useCommonWords=args.common_words, + useLeet=args.leet, + useLocales=args.locales, +- extraWords=extraWords ++ extraWords=extraWords, ++ outputFile=args.outputFile + ) + + # Urging users to place an issue on Github... diff --git a/debian/patches/initalize-config.patch b/debian/patches/initalize-config.patch index 6f24802..10f67a9 100644 --- a/debian/patches/initalize-config.patch +++ b/debian/patches/initalize-config.patch @@ -2,67 +2,55 @@ Upstream creates and initializes the ~/.config/OSRFramework in the setup.py. We disable this for Kali and create a function initialize_config_files to manage the creation and installation of all - the required files, including maltego settings. + the required files. Author: Sophie Brun Last-Update: 2017-09-18 --- This patch header follows DEP-3: http://dep.debian.net/deps/dep3/ --- a/osrframework/utils/configuration.py +++ b/osrframework/utils/configuration.py -@@ -22,8 +22,52 @@ - +@@ -23,9 +23,40 @@ + import ConfigParser import os import sys +import shutil +import glob import osrframework.utils.errors as errors -+ + +def initialize_config_files(): + """ + Function to create and initialize the configuration files and + directories. + """ + LOCAL_CONFIG_DIR = getConfigPath()["appPath"] -+ LOCAL_CONFIG_TRANSFORMS_DIR = getConfigPath()["appPathTransforms"] ++# LOCAL_CONFIG_TRANSFORMS_DIR = getConfigPath()["appPathTransforms"] + INSTALL_PATH = '/usr/lib/python2.7/dist-packages/osrframework' + + # Copy default config file into osrframework local config -+ for dirconf in ["default", "plugins", "server"]: ++ for dirconf in ["default", "plugins"]: + shutil.copytree(os.path.join('/etc/osrframework',dirconf), os.path.join(LOCAL_CONFIG_DIR, dirconf)) + + # Create directory data + os.mkdir(getConfigPath()["appPathData"]) + + # Create directory Transforms before copy *.py files -+ os.mkdir(LOCAL_CONFIG_TRANSFORMS_DIR) ++# os.mkdir(LOCAL_CONFIG_TRANSFORMS_DIR) + + # Copy *.py files in transforms directory -+ for f in glob.glob(os.path.join(INSTALL_PATH, "transforms/[a-z]*.py")): -+ shutil.copy(f, LOCAL_CONFIG_TRANSFORMS_DIR) -+ for f in ["alias_generator.py", "entify.py", "phonefy.py", "searchfy.py", "mailfy.py", "usufy.py", "domainfy.py"]: -+ shutil.copy((os.path.join(INSTALL_PATH, f)), LOCAL_CONFIG_TRANSFORMS_DIR) ++# for f in glob.glob(os.path.join(INSTALL_PATH, "transforms/[a-z]*.py")): ++# shutil.copy(f, LOCAL_CONFIG_TRANSFORMS_DIR) ++# for f in ["alias_generator.py", "entify.py", "phonefy.py", "searchfy.py", "mailfy.py", "usufy.py", "domainfy.py", "enumerate-profiles.py"]: ++# shutil.copy((os.path.join(INSTALL_PATH, f)), LOCAL_CONFIG_TRANSFORMS_DIR) + + # Copy default config files for the user + for f in ["browser.cfg", "general.cfg"]: + shutil.copy((os.path.join(getConfigPath()["appPathDefaults"], f)),LOCAL_CONFIG_DIR) + -+ # Configure maltego -+ LOCAL_CONFIG_DEFAULT_DIR = getConfigPath()["appPathDefaults"] -+ try: -+ import osrframework.transforms.lib.configure_maltego as maltego -+ maltego.configureMaltego(transformsConfigFolder=LOCAL_CONFIG_TRANSFORMS_DIR, -+ base=os.path.join(LOCAL_CONFIG_DEFAULT_DIR, "osrframework-maltego-settings"), -+ debug=False, backupPath=LOCAL_CONFIG_DEFAULT_DIR) -+ except Exception, e: -+ print "[!] The Maltego configuration file to use i3visio transforms could not be created and thus, cannot be used. Check the following error:" -+ print str(e) -+ -+ def changePermissionsRecursively(path, uid, gid): """ - Function to recursively change the user id and group id. It sets 700 -@@ -84,9 +128,10 @@ def getConfigPath(configFileName = None) + Function to recursively change the user id and group id. +@@ -92,9 +123,10 @@ def getConfigPath(configFileName = None) } # Creating them if they don't exist @@ -76,8 +64,8 @@ return paths -@@ -100,6 +145,8 @@ def returnListOfConfigurationValues(util - ''' +@@ -117,6 +149,8 @@ def returnListOfConfigurationValues(util + """ VALUES = {} + if not os.path.exists(getConfigPath()["appPath"]): diff --git a/debian/patches/series b/debian/patches/series index ddbfffa..00fdd9b 100644 --- a/debian/patches/series +++ b/debian/patches/series @@ -1,10 +1,7 @@ -fix-syntax-error.patch cleanup-setup.patch dont-use-validate-email.patch drop-explicit-need-of-importlib.patch upgrade-installation.patch -fix-exit-function.patch initalize-config.patch add-script-config-upgrade.patch -drop-check-updates.patch -create-real-temporary-dir.patch +fix-alias-generator.patch diff --git a/debian/patches/upgrade-installation.patch b/debian/patches/upgrade-installation.patch index 6043b36..eda2c8c 100644 --- a/debian/patches/upgrade-installation.patch +++ b/debian/patches/upgrade-installation.patch @@ -10,19 +10,18 @@ This patch header follows DEP-3: http://dep.debian.net/deps/dep3/ --- a/setup.py +++ b/setup.py -@@ -22,6 +22,8 @@ +@@ -23,6 +23,7 @@ import os import sys +import glob -+ - from setuptools import setup - - # Checking if obsolete versions are installed in the machine -@@ -179,6 +181,11 @@ setup( - "osrframework.searchengines", - "osrframework.domains", + from setuptools import setup, find_packages + import shutil + import site +@@ -168,6 +169,11 @@ setup( + 'Topic :: Text Processing :: Markup :: HTML' ], + packages=find_packages(), + data_files=[ + ("/etc/osrframework/default", glob.glob("config/*.cfg")), + ("/etc/osrframework/plugins/patterns", ["config/plugins/pattern.py.sample"]), diff --git a/debian/pydist-overrides b/debian/pydist-overrides index 9459ab7..7d7fcee 100644 --- a/debian/pydist-overrides +++ b/debian/pydist-overrides @@ -1,4 +1,3 @@ -Skype4Py python-skype4py python_whois python-whois pyexcel python-pyexcel pyexcel_ods python-pyexcel-ods